From b7aafca7b708b40cdfbeeff3798e2074722ece7e Mon Sep 17 00:00:00 2001
From: Balasubramanian Kandasamy <balasubramanian.kandasamy@oracle.com>
Date: Mon, 26 Feb 2018 15:55:16 +0530
Subject: [PATCH 001/164] Raise version number after cloning 5.5.60

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 44f719ca097..23e938e60e1 100644
--- a/VERSION
+++ b/VERSION
@@ -1,4 +1,4 @@
 MYSQL_VERSION_MAJOR=5
 MYSQL_VERSION_MINOR=5
-MYSQL_VERSION_PATCH=60
+MYSQL_VERSION_PATCH=61
 MYSQL_VERSION_EXTRA=

From d4e94739231be30cbda7f910eb1d76cfba2e549b Mon Sep 17 00:00:00 2001
From: Faustin Lammler <faustin@falared.net>
Date: Tue, 3 Apr 2018 10:33:11 -0300
Subject: [PATCH 002/164] Package dependency case problem

---
 debian/control | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/control b/debian/control
index 862cd4f3cea..79ccf8c8fd4 100644
--- a/debian/control
+++ b/debian/control
@@ -373,10 +373,10 @@ Description: MariaDB database regression test suite (metapackage depending on th
 Package: mariadb-connect-engine-10.0
 Section: database
 Architecture: any
-Depends: libxml2, mariadb-server-10.0|mariadb-galera-server-10.0, unixODBC
+Depends: libxml2, mariadb-server-10.0|mariadb-galera-server-10.0, unixodbc
 Build-depends: libxml2-dev,
                mariadb-server-10.0|mariadb-galera-server-10.0,
-               unixODBC-dev
+               unixodbc-dev
 Description: Connect storage engine for MariaDB
  Connect engine supports a number of file formats (dbf, xml, txt, bin, etc),
  connections to ODBC tables and remote MySQL tables, as well as a number of

From d982e717aba67227ec40761a21a4211db91aa0e2 Mon Sep 17 00:00:00 2001
From: Arun Kuruvila <arun.kuruvila@oracle.com>
Date: Mon, 9 Apr 2018 11:18:12 +0530
Subject: [PATCH 003/164] Bug#27510150: MYSQLDUMP FAILS FOR SPECIFIC --WHERE
 CLAUSES

Description: Mysqldump utility fails for specific clauses
used with the option, 'where'.

Analysis:- Method, "fix_identifier_with_newline()" that
prefixes all occurrences of newline char ('\n') in incoming
buffer does not verify the size of the buffer. The buffer in
which the incoming buffer is copied is limited to 2048 bytes
and the method does not try to allocate additional memory
for larger incoming buffers.

Fix:- Method, "fix_identifier_with_newline()" is modified
to fix this issue.
---
 client/mysqldump.c | 182 +++++++++++++++++++++++++++++++++++----------
 1 file changed, 142 insertions(+), 40 deletions(-)

diff --git a/client/mysqldump.c b/client/mysqldump.c
index 2775769fe52..34037337091 100644
--- a/client/mysqldump.c
+++ b/client/mysqldump.c
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2000, 2017, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -549,7 +549,8 @@ static int dump_tablespaces_for_databases(char** databases);
 static int dump_tablespaces(char* ts_where);
 static void print_comment(FILE *sql_file, my_bool is_error, const char *format,
                           ...);
-static const char* fix_identifier_with_newline(char*);
+static char const* fix_identifier_with_newline(char const* object_name,
+                                               my_bool* freemem);
 
 
 /*
@@ -644,13 +645,19 @@ static void write_header(FILE *sql_file, char *db_name)
   }
   else if (!opt_compact)
   {
+    my_bool freemem= FALSE;
+    char const* text= fix_identifier_with_newline(db_name, &freemem);
+
     print_comment(sql_file, 0,
                   "-- MySQL dump %s  Distrib %s, for %s (%s)\n--\n",
                   DUMP_VERSION, MYSQL_SERVER_VERSION, SYSTEM_TYPE,
                   MACHINE_TYPE);
     print_comment(sql_file, 0, "-- Host: %s    Database: %s\n",
                   current_host ? current_host : "localhost",
-                  db_name ? fix_identifier_with_newline(db_name) : "");
+                  text);
+    if (freemem)
+      my_free((void*)text);
+
     print_comment(sql_file, 0,
                   "-- ------------------------------------------------------\n"
                  );
@@ -1982,31 +1989,84 @@ static void print_comment(FILE *sql_file, my_bool is_error, const char *format,
   print_xml_comment(sql_file, strlen(comment_buff), comment_buff);
 }
 
-/*
- This function accepts object names and prefixes -- wherever \n
- character is found.
 
- @param[in]     object_name
+/**
+  @brief Accepts object names and prefixes them with "-- " wherever
+         end-of-line character ('\n') is found.
 
- @return
-    @retval fixed object name.
+  @param[in]  object_name   object name list (concatenated string)
+  @param[out] freemem       should buffer be released after usage
+
+  @return
+    @retval                 pointer to a string with prefixed objects
 */
-
-static const char* fix_identifier_with_newline(char* object_name)
+static char const* fix_identifier_with_newline(char const* object_name, my_bool* freemem)
 {
-  static char buff[COMMENT_LENGTH]= {0};
-  char *ptr= buff;
-  memset(buff, 0, 255);
-  while(*object_name)
+  const size_t PREFIX_LENGTH= 3; // strlen ("-- ")
+
+  // static buffer for replacement procedure
+  static char storage[NAME_LEN + 1];
+  static char* buffer= storage;
+  static size_t buffer_size= sizeof(storage) - 1;
+  size_t index= 0;
+  size_t required_size= 0;
+
+  // we presume memory allocation won't be needed
+  *freemem= FALSE;
+
+  // traverse and reformat objects
+  while (object_name && *object_name)
   {
-    *ptr++ = *object_name;
+    ++required_size;
     if (*object_name == '\n')
-      ptr= strmov(ptr, "-- ");
-    object_name++;
+      required_size+= PREFIX_LENGTH;
+
+    // do we need dynamic (re)allocation
+    if (required_size > buffer_size)
+    {
+      // new alloc size increased in COMMENT_LENGTH multiple
+      buffer_size= COMMENT_LENGTH * (1 + required_size/COMMENT_LENGTH);
+
+      // is our buffer already dynamically allocated
+      if (*freemem)
+      {
+        // just realloc
+        buffer= (char*)my_realloc(buffer, buffer_size + 1, MYF(MY_WME));
+        if (!buffer)
+          exit(1);
+      }
+      else
+      {
+        // dynamic allocation + copy from static buffer
+        buffer= (char*)my_malloc(buffer_size + 1, MYF(MY_WME));
+        if (!buffer)
+          exit(1);
+
+        strncpy(buffer, storage, index);
+        *freemem= TRUE;
+      }
+    }
+
+    // copy a character
+    buffer[index]= *object_name;
+    ++index;
+
+    // prefix new lines with double dash
+    if (*object_name == '\n')
+    {
+      strcpy(buffer + index, "-- ");
+      index += PREFIX_LENGTH;
+    }
+
+    ++object_name;
   }
-  return buff;
+
+  // don't forget null termination
+  buffer[index]= '\0';
+  return buffer;
 }
 
+
 /*
  create_delimiter
  Generate a new (null-terminated) string that does not exist in  query 
@@ -2066,6 +2126,8 @@ static uint dump_events_for_db(char *db)
 
   char       db_cl_name[MY_CS_NAME_SIZE];
   int        db_cl_altered= FALSE;
+  my_bool    freemem= FALSE;
+  char const *text= fix_identifier_with_newline(db, &freemem);
 
   DBUG_ENTER("dump_events_for_db");
   DBUG_PRINT("enter", ("db: '%s'", db));
@@ -2075,7 +2137,9 @@ static uint dump_events_for_db(char *db)
   /* nice comments */
   print_comment(sql_file, 0,
                 "\n--\n-- Dumping events for database '%s'\n--\n",
-                fix_identifier_with_newline(db));
+                text);
+  if (freemem)
+    my_free((void*)text);
 
   /*
     not using "mysql_query_with_error_report" because we may have not
@@ -2284,6 +2348,8 @@ static uint dump_routines_for_db(char *db)
 
   char       db_cl_name[MY_CS_NAME_SIZE];
   int        db_cl_altered= FALSE;
+  my_bool    freemem= FALSE;
+  char const *text= fix_identifier_with_newline(db, &freemem);
 
   DBUG_ENTER("dump_routines_for_db");
   DBUG_PRINT("enter", ("db: '%s'", db));
@@ -2292,8 +2358,10 @@ static uint dump_routines_for_db(char *db)
 
   /* nice comments */
   print_comment(sql_file, 0,
-                "\n--\n-- Dumping routines for database '%s'\n--\n",
-                fix_identifier_with_newline(db));
+                "\n--\n-- Dumping routines for database '%s'\n--\n", text);
+
+  if (freemem)
+    my_free((void*)text);
 
   /*
     not using "mysql_query_with_error_report" because we may have not
@@ -2348,11 +2416,17 @@ static uint dump_routines_for_db(char *db)
                              row[2] ? strlen(row[2]) : 0));
           if (row[2] == NULL)
           {
+            my_bool freemem= FALSE;
+            char const* text= fix_identifier_with_newline(current_user, &freemem);
+
             print_comment(sql_file, 1, "\n-- insufficient privileges to %s\n",
                           query_buff);
             print_comment(sql_file, 1,
                           "-- does %s have permissions on mysql.proc?\n\n",
-                          fix_identifier_with_newline(current_user));
+                          text);
+            if (freemem)
+              my_free((void*)text);
+
             maybe_die(EX_MYSQLERR,"%s has insufficent privileges to %s!", current_user, query_buff);
           }
           else if (strlen(row[2]))
@@ -2547,6 +2621,8 @@ static uint get_table_structure(char *table, char *db, char *table_type,
       /* Make an sql-file, if path was given iow. option -T was given */
       char buff[20+FN_REFLEN];
       MYSQL_FIELD *field;
+      my_bool freemem= FALSE;
+      char const *text;
 
       my_snprintf(buff, sizeof(buff), "show create table %s", result_table);
 
@@ -2563,14 +2639,17 @@ static uint get_table_structure(char *table, char *db, char *table_type,
         write_header(sql_file, db);
       }
 
+      text= fix_identifier_with_newline(result_table, &freemem);
       if (strcmp (table_type, "VIEW") == 0)         /* view */
         print_comment(sql_file, 0,
                       "\n--\n-- Temporary table structure for view %s\n--\n\n",
-                      fix_identifier_with_newline(result_table));
+                      text);
       else
         print_comment(sql_file, 0,
-                      "\n--\n-- Table structure for table %s\n--\n\n",
-                      fix_identifier_with_newline(result_table));
+                      "\n--\n-- Table structure for table %s\n--\n\n", text);
+
+      if (freemem)
+        my_free((void*)text);
 
       if (opt_drop)
       {
@@ -2803,6 +2882,9 @@ static uint get_table_structure(char *table, char *db, char *table_type,
     /* Make an sql-file, if path was given iow. option -T was given */
     if (!opt_no_create_info)
     {
+      my_bool freemem= FALSE;
+      char const *text;
+
       if (path)
       {
         if (!(sql_file= open_sql_file_for_table(table, O_WRONLY)))
@@ -2810,9 +2892,13 @@ static uint get_table_structure(char *table, char *db, char *table_type,
         write_header(sql_file, db);
       }
 
+      text= fix_identifier_with_newline(result_table, &freemem);
       print_comment(sql_file, 0,
                     "\n--\n-- Table structure for table %s\n--\n\n",
-                    fix_identifier_with_newline(result_table));
+                    text);
+      if (freemem)
+        my_free((void*)text);
+
       if (opt_drop)
         fprintf(sql_file, "DROP TABLE IF EXISTS %s;\n", result_table);
       if (!opt_xml)
@@ -3515,25 +3601,34 @@ static void dump_table(char *table, char *db)
   }
   else
   {
-    print_comment(md_result_file, 0,
-                  "\n--\n-- Dumping data for table %s\n--\n",
-                  fix_identifier_with_newline(result_table));
-    
+    my_bool freemem= FALSE;
+    char const* text= fix_identifier_with_newline(result_table, &freemem);
+    print_comment(md_result_file, 0, "\n--\n-- Dumping data for table %s\n--\n",
+                  text);
+    if (freemem)
+      my_free((void*)text);
+
     dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ * FROM ");
     dynstr_append_checked(&query_string, result_table);
 
     if (where)
     {
-      print_comment(md_result_file, 0, "-- WHERE:  %s\n",
-        fix_identifier_with_newline(where));
+      freemem= FALSE;
+      text= fix_identifier_with_newline(where, &freemem);
+      print_comment(md_result_file, 0, "-- WHERE:  %s\n", text);
+      if (freemem)
+        my_free((void*)text);
 
       dynstr_append_checked(&query_string, " WHERE ");
       dynstr_append_checked(&query_string, where);
     }
     if (order_by)
     {
-      print_comment(md_result_file, 0, "-- ORDER BY:  %s\n",
-        fix_identifier_with_newline(order_by));
+      freemem= FALSE;
+      text= fix_identifier_with_newline(order_by, &freemem);
+      print_comment(md_result_file, 0, "-- ORDER BY:  %s\n", text);
+      if (freemem)
+        my_free((void*)text);
 
       dynstr_append_checked(&query_string, " ORDER BY ");
       dynstr_append_checked(&query_string, order_by);
@@ -4302,10 +4397,13 @@ static int init_dumping(char *database, int init_func(char*))
       */
       char quoted_database_buf[NAME_LEN*2+3];
       char *qdatabase= quote_name(database,quoted_database_buf,opt_quoted);
+      my_bool freemem= FALSE;
+      char const* text= fix_identifier_with_newline(qdatabase, &freemem);
 
-      print_comment(md_result_file, 0,
-                    "\n--\n-- Current Database: %s\n--\n",
-                    fix_identifier_with_newline(qdatabase));
+      print_comment(md_result_file, 0, "\n--\n-- Current Database: %s\n--\n",
+                    text);
+      if (freemem)
+        my_free((void*)text);
 
       /* Call the view or table specific function */
       init_func(qdatabase);
@@ -5265,6 +5363,8 @@ static my_bool get_view_structure(char *table, char* db)
   char       table_buff2[NAME_LEN*2+3];
   char       query[QUERY_LENGTH];
   FILE       *sql_file= md_result_file;
+  my_bool    freemem= FALSE;
+  char const *text;
   DBUG_ENTER("get_view_structure");
 
   if (opt_no_create_info) /* Don't write table creation info */
@@ -5309,9 +5409,11 @@ static my_bool get_view_structure(char *table, char* db)
     write_header(sql_file, db);
   }
 
+  text= fix_identifier_with_newline(result_table, &freemem);
   print_comment(sql_file, 0,
-                "\n--\n-- Final view structure for view %s\n--\n\n",
-                fix_identifier_with_newline(result_table));
+                "\n--\n-- Final view structure for view %s\n--\n\n", text);
+  if (freemem)
+    my_free((void*)text);
 
   /* Table might not exist if this view was dumped with --tab. */
   fprintf(sql_file, "/*!50001 DROP TABLE IF EXISTS %s*/;\n", opt_quoted_table);

From 940b88b686bcf037a36f6e81be4017a07fcf782a Mon Sep 17 00:00:00 2001
From: Ajo Robert <ajo.robert@oracle.com>
Date: Tue, 10 Apr 2018 00:30:59 +0530
Subject: [PATCH 004/164] Bug#27197235 USER VARIABLE + UINON + DECIMAL COLUMN
 RETURNS                           WRONG VALUES

User variables will have the default session collation
associated with it. And a select which uses it as part of a
union may infer the collation while type merging.
This leads to problems when the result is of DECIMAL type.
Setting the appropriate collation of DECIMAL result type
is missing in 5.7 code base.

Added code to set appropriate collation when the result is
of DECIMAL type during Item_type_holder::join_types().
---
 mysql-test/r/union.result | 16 ++++++++++++++++
 mysql-test/t/union.test   | 14 ++++++++++++++
 sql/item.cc               |  3 ++-
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/mysql-test/r/union.result b/mysql-test/r/union.result
index 97c1a8b4d49..c0b364d0fde 100644
--- a/mysql-test/r/union.result
+++ b/mysql-test/r/union.result
@@ -1860,3 +1860,19 @@ DROP TABLE t17059925, t2, t3;
 SET @@long_query_time= @old_long_query_time;
 SET @@global.log_output= @old_log_output;
 SET @@global.slow_query_log= @old_slow_query_log;
+#
+# Bug#27197235 USER VARIABLE + UINON + DECIMAL COLUMN RETURNS
+#              WRONG VALUES
+#
+SET NAMES utf8;
+SET @advertAcctId = 1000003;
+select @advertAcctId as a from dual union all select 1.0 from dual;
+a
+1000003.0
+1.0
+SET NAMES latin1;
+SET @advertAcctId = 1000003;
+select @advertAcctId as a from dual union all select 1.0 from dual;
+a
+1000003.0
+1.0
diff --git a/mysql-test/t/union.test b/mysql-test/t/union.test
index 147c1d3834b..d7e362558e3 100644
--- a/mysql-test/t/union.test
+++ b/mysql-test/t/union.test
@@ -1256,3 +1256,17 @@ SET @@long_query_time= @old_long_query_time;
 SET @@global.log_output= @old_log_output;
 SET @@global.slow_query_log= @old_slow_query_log;
 
+--echo #
+--echo # Bug#27197235 USER VARIABLE + UINON + DECIMAL COLUMN RETURNS
+--echo #              WRONG VALUES
+--echo #
+
+let $old_charset= `SELECT @@character_set_client`;
+
+SET NAMES utf8;
+SET @advertAcctId = 1000003;
+select @advertAcctId as a from dual union all select 1.0 from dual;
+
+eval SET NAMES $old_charset;
+SET @advertAcctId = 1000003;
+select @advertAcctId as a from dual union all select 1.0 from dual;
diff --git a/sql/item.cc b/sql/item.cc
index a37a61453e8..07d64881eeb 100644
--- a/sql/item.cc
+++ b/sql/item.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2000, 2017, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -8266,6 +8266,7 @@ bool Item_type_holder::join_types(THD *thd, Item *item)
   }
   if (Field::result_merge_type(fld_type) == DECIMAL_RESULT)
   {
+    collation.set_numeric();
     decimals= min(max(decimals, item->decimals), DECIMAL_MAX_SCALE);
     int item_int_part= item->decimal_int_part();
     int item_prec = max(prev_decimal_int_part, item_int_part) + decimals;

From a08508abf83d953f11e4823798398d9229ba0237 Mon Sep 17 00:00:00 2001
From: Arun Kuruvila <arun.kuruvila@oracle.com>
Date: Tue, 24 Apr 2018 10:02:04 +0530
Subject: [PATCH 005/164] Bug#27407480: AUTOMATIC_SP_PRIVILEGES REQUIRES NEED
 THE               INSERT PRIVILEGES FOR MYSQL.USER TABLE

Description:- Incorrect granting of EXECUTE and ALTER
ROUTINE privileges when the 'automatic_sp_privileges'
variable is set.

Fix:- EXECUTE and ALTER ROUTINE privileges are correctly
granted to the creator of the procedure when the
'automatic_sp_privileges' is SET.
---
 mysql-test/r/grant.result |  3 +--
 sql/sql_acl.cc            | 15 ++++-----------
 2 files changed, 5 insertions(+), 13 deletions(-)

diff --git a/mysql-test/r/grant.result b/mysql-test/r/grant.result
index 7603e4b1b1c..e524bb5cac2 100644
--- a/mysql-test/r/grant.result
+++ b/mysql-test/r/grant.result
@@ -1780,8 +1780,6 @@ BEGIN
 SET @x = 0;
 REPEAT SET @x = @x + 1; UNTIL @x > p1 END REPEAT;
 END ;||
-Warnings:
-Warning	1404	Failed to grant EXECUTE and ALTER ROUTINE privileges
 SHOW GRANTS FOR 'user1'@'localhost';
 Grants for user1@localhost
 GRANT USAGE ON *.* TO 'user1'@'localhost'
@@ -1791,6 +1789,7 @@ SHOW GRANTS FOR 'user2';
 Grants for user2@%
 GRANT USAGE ON *.* TO 'user2'@'%'
 GRANT CREATE, CREATE ROUTINE ON `db1`.* TO 'user2'@'%'
+GRANT EXECUTE, ALTER ROUTINE ON PROCEDURE `db1`.`proc2` TO 'user2'@'%'
 DROP PROCEDURE db1.proc1;
 DROP PROCEDURE db1.proc2;
 REVOKE ALL ON db1.* FROM 'user1'@'localhost';
diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc
index 385484f2b01..705b4792a37 100644
--- a/sql/sql_acl.cc
+++ b/sql/sql_acl.cc
@@ -1,4 +1,4 @@
-/* Copyright (c) 2000, 2017, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -7670,19 +7670,12 @@ bool sp_grant_privileges(THD *thd, const char *sp_db, const char *sp_name,
   if (!(combo=(LEX_USER*) thd->alloc(sizeof(st_lex_user))))
     DBUG_RETURN(TRUE);
 
-  combo->user.str= sctx->user;
+  combo->user.str= (char *) sctx->priv_user;
 
   mysql_mutex_lock(&acl_cache->lock);
 
-  if ((au= find_acl_user(combo->host.str=(char*)sctx->host_or_ip,combo->user.str,FALSE)))
-    goto found_acl;
-  if ((au= find_acl_user(combo->host.str=(char*)sctx->get_host()->ptr(),
-                         combo->user.str,FALSE)))
-    goto found_acl;
-  if ((au= find_acl_user(combo->host.str=(char*)sctx->get_ip()->ptr(),
-                         combo->user.str,FALSE)))
-    goto found_acl;
-  if((au= find_acl_user(combo->host.str=(char*)"%", combo->user.str, FALSE)))
+ if ((au= find_acl_user(combo->host.str= (char *) sctx->priv_host,
+                        combo->user.str, FALSE)))
     goto found_acl;
 
   mysql_mutex_unlock(&acl_cache->lock);

From 6d570d729682039edd6c490187a0434e7d75d486 Mon Sep 17 00:00:00 2001
From: Arun Kuruvila <arun.kuruvila@oracle.com>
Date: Thu, 10 May 2018 10:14:30 +0530
Subject: [PATCH 006/164] Bug#27230925: HANDLE_FATAL_SIGNAL (SIG=11) IN        
       SHOW_ROUTINE_GRANTS

Description :- Server crashes in show_routine_grants().

Analysis :- When "grant_reload_procs_priv" encounters
an error, the grant structures (structures with column,
function and procedure privileges) are freed. Server
crashes when trying to access these structures later.

Fix :- Grant structures are retained even when
"grant_reload_procs_priv()" encounters an error while
reloading column, function and procedure privileges.
---
 mysql-test/r/grant.result |  1 +
 mysql-test/t/grant.test   |  3 ++
 sql/sql_acl.cc            | 92 ++++++++++++++++++++++++++-------------
 3 files changed, 65 insertions(+), 31 deletions(-)

diff --git a/mysql-test/r/grant.result b/mysql-test/r/grant.result
index e524bb5cac2..d151b58e9ce 100644
--- a/mysql-test/r/grant.result
+++ b/mysql-test/r/grant.result
@@ -1692,6 +1692,7 @@ revoke create, insert on mysqltest.t6 from mysqltest@localhost;
 drop user mysqltest@localhost;
 drop database mysqltest;
 use test;
+call mtr.add_suppression("Can't open and lock privilege tables");
 FLUSH PRIVILEGES without procs_priv table.
 RENAME TABLE mysql.procs_priv TO mysql.procs_gone;
 FLUSH PRIVILEGES;
diff --git a/mysql-test/t/grant.test b/mysql-test/t/grant.test
index 1e1cb6c24dc..2a1a8e34efb 100644
--- a/mysql-test/t/grant.test
+++ b/mysql-test/t/grant.test
@@ -1667,6 +1667,9 @@ use test;
 #
 # Bug#16470 crash on grant if old grant tables
 #
+
+call mtr.add_suppression("Can't open and lock privilege tables");
+
 --echo FLUSH PRIVILEGES without procs_priv table.
 RENAME TABLE mysql.procs_priv TO mysql.procs_gone;
 --error ER_NO_SUCH_TABLE
diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc
index 705b4792a37..64888f7308a 100644
--- a/sql/sql_acl.cc
+++ b/sql/sql_acl.cc
@@ -4871,6 +4871,7 @@ end_index_init:
     exists.
 
   @param thd A pointer to the thread handler object.
+  @param table A pointer to the table list.
 
   @see grant_reload
 
@@ -4879,31 +4880,22 @@ end_index_init:
     @retval TRUE An error has occurred.
 */
 
-static my_bool grant_reload_procs_priv(THD *thd)
+static my_bool grant_reload_procs_priv(THD *thd, TABLE_LIST *table)
 {
   HASH old_proc_priv_hash, old_func_priv_hash;
-  TABLE_LIST table;
   my_bool return_val= FALSE;
   DBUG_ENTER("grant_reload_procs_priv");
 
-  table.init_one_table("mysql", 5, "procs_priv",
-                       strlen("procs_priv"), "procs_priv",
-                       TL_READ);
-  table.open_type= OT_BASE_ONLY;
-
-  if (open_and_lock_tables(thd, &table, FALSE, MYSQL_LOCK_IGNORE_TIMEOUT))
-    DBUG_RETURN(TRUE);
-
-  mysql_rwlock_wrlock(&LOCK_grant);
   /* Save a copy of the current hash if we need to undo the grant load */
   old_proc_priv_hash= proc_priv_hash;
   old_func_priv_hash= func_priv_hash;
 
-  if ((return_val= grant_load_procs_priv(table.table)))
+  if ((return_val= grant_load_procs_priv(table->table)))
   {
     /* Error; Reverting to old hash */
     DBUG_PRINT("error",("Reverting to old privileges"));
-    grant_free();
+    my_hash_free(&proc_priv_hash);
+    my_hash_free(&func_priv_hash);
     proc_priv_hash= old_proc_priv_hash;
     func_priv_hash= old_func_priv_hash;
   }
@@ -4912,9 +4904,7 @@ static my_bool grant_reload_procs_priv(THD *thd)
     my_hash_free(&old_proc_priv_hash);
     my_hash_free(&old_func_priv_hash);
   }
-  mysql_rwlock_unlock(&LOCK_grant);
 
-  close_mysql_tables(thd);
   DBUG_RETURN(return_val);
 }
 
@@ -4936,7 +4926,7 @@ static my_bool grant_reload_procs_priv(THD *thd)
 
 my_bool grant_reload(THD *thd)
 {
-  TABLE_LIST tables[2];
+  TABLE_LIST tables[3];
   HASH old_column_priv_hash;
   MEM_ROOT old_mem;
   my_bool return_val= 1;
@@ -4952,15 +4942,57 @@ my_bool grant_reload(THD *thd)
   tables[1].init_one_table(C_STRING_WITH_LEN("mysql"),
                            C_STRING_WITH_LEN("columns_priv"),
                            "columns_priv", TL_READ);
+  tables[2].init_one_table(C_STRING_WITH_LEN("mysql"),
+                           C_STRING_WITH_LEN("procs_priv"),
+                           "procs_priv", TL_READ);
+
   tables[0].next_local= tables[0].next_global= tables+1;
-  tables[0].open_type= tables[1].open_type= OT_BASE_ONLY;
+  tables[1].next_local= tables[1].next_global= tables+2;
+  tables[0].open_type= tables[1].open_type= tables[2].open_type= OT_BASE_ONLY;
+
+  /*
+    Reload will work in the following manner:-
+
+                             proc_priv_hash structure
+                              /                     \
+                    not initialized                 initialized
+                   /               \                     |
+    mysql.procs_priv table        Server Startup         |
+        is missing                      \                |
+             |                         open_and_lock_tables()
+    Assume we are working on           /success             \failure
+    pre 4.1 system tables.        Normal Scenario.          An error is thrown.
+    A warning is printed          Reload column privilege.  Retain the old hash.
+    and continue with             Reload function and
+    reloading the column          procedure privileges,
+    privileges.                   if available.
+  */
+
+  if (!(my_hash_inited(&proc_priv_hash)))
+    tables[2].open_strategy= TABLE_LIST::OPEN_IF_EXISTS;
 
   /*
     To avoid deadlocks we should obtain table locks before
     obtaining LOCK_grant rwlock.
   */
   if (open_and_lock_tables(thd, tables, FALSE, MYSQL_LOCK_IGNORE_TIMEOUT))
+  {
+    if (thd->stmt_da->is_error())
+    {
+      sql_print_error("Fatal error: Can't open and lock privilege tables: %s",
+                      thd->stmt_da->message());
+    }
     goto end;
+  }
+
+  if (tables[2].table == NULL)
+  {
+    sql_print_warning("Table 'mysql.procs_priv' does not exist. "
+                      "Please run mysql_upgrade.");
+    push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, ER_NO_SUCH_TABLE,
+                        ER(ER_NO_SUCH_TABLE), tables[2].db,
+                        tables[2].table_name);
+  }
 
   mysql_rwlock_wrlock(&LOCK_grant);
   old_column_priv_hash= column_priv_hash;
@@ -4972,10 +5004,18 @@ my_bool grant_reload(THD *thd)
   old_mem= memex;
   init_sql_alloc(&memex, ACL_ALLOC_BLOCK_SIZE, 0);
 
-  if ((return_val= grant_load(thd, tables)))
+  /*
+    tables[2].table i.e. procs_priv can be null if we are working with
+    pre 4.1 privilage tables
+  */
+  if ((return_val= (grant_load(thd, tables) ||
+                    (tables[2].table != NULL &&
+                     grant_reload_procs_priv(thd, &tables[2])))
+     ))
   {						// Error. Revert to old hash
     DBUG_PRINT("error",("Reverting to old privileges"));
-    grant_free();				/* purecov: deadcode */
+    my_hash_free(&column_priv_hash);
+    free_root(&memex,MYF(0));
     column_priv_hash= old_column_priv_hash;	/* purecov: deadcode */
     memex= old_mem;				/* purecov: deadcode */
   }
@@ -4983,22 +5023,12 @@ my_bool grant_reload(THD *thd)
   {
     my_hash_free(&old_column_priv_hash);
     free_root(&old_mem,MYF(0));
+    grant_version++;
   }
   mysql_rwlock_unlock(&LOCK_grant);
-  close_mysql_tables(thd);
-
-  /*
-    It is OK failing to load procs_priv table because we may be
-    working with 4.1 privilege tables.
-  */
-  if (grant_reload_procs_priv(thd))
-    return_val= 1;
-
-  mysql_rwlock_wrlock(&LOCK_grant);
-  grant_version++;
-  mysql_rwlock_unlock(&LOCK_grant);
 
 end:
+  close_mysql_tables(thd);
   DBUG_RETURN(return_val);
 }
 

From bbc2e37fe4e0ca3a7cfa1437a763dc43829e98e2 Mon Sep 17 00:00:00 2001
From: Arun Kuruvila <arun.kuruvila@oracle.com>
Date: Mon, 14 May 2018 11:28:13 +0530
Subject: [PATCH 007/164] Bug#27759871: BACKRONYM ISSUE IS STILL IN MYSQL 5.7

Description:- Client applications establishes connection to
server, which does not support SSL, via TCP even when SSL is
enforced via MYSQL_OPT_SSL_MODE or MYSQL_OPT_SSL_ENFORCE or
MYSQL_OPT_SSL_VERIFY_SERVER_CERT.

Analysis:- There exist no error handling for catching client
applications which enforces SSL connection to connect to a
server which does not support SSL.

Fix:- Error handling is done to catch above mentioned
scenarios.
---
 include/sql_common.h  |  5 ++++-
 libmysqld/libmysqld.c |  5 ++++-
 sql-common/client.c   | 41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/include/sql_common.h b/include/sql_common.h
index 45e90d438fb..9571dff9778 100644
--- a/include/sql_common.h
+++ b/include/sql_common.h
@@ -1,7 +1,7 @@
 #ifndef SQL_COMMON_INCLUDED
 #define SQL_COMMON_INCLUDED
 
-/* Copyright (c) 2003, 2017, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
    
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -96,6 +96,9 @@ void set_stmt_error(MYSQL_STMT *stmt, int errcode, const char *sqlstate,
 void set_mysql_error(MYSQL *mysql, int errcode, const char *sqlstate);
 void set_mysql_extended_error(MYSQL *mysql, int errcode, const char *sqlstate,
                               const char *format, ...);
+#ifdef EMBEDDED_LIBRARY
+int embedded_ssl_check(MYSQL *mysql);
+#endif
 
 /* client side of the pluggable authentication */
 struct st_plugin_vio_info;
diff --git a/libmysqld/libmysqld.c b/libmysqld/libmysqld.c
index 6de1e3383d2..85ca0cf4bd8 100644
--- a/libmysqld/libmysqld.c
+++ b/libmysqld/libmysqld.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -173,6 +173,9 @@ mysql_real_connect(MYSQL *mysql,const char *host, const char *user,
   if (db)
     client_flag|=CLIENT_CONNECT_WITH_DB;
 
+  if (embedded_ssl_check(mysql))
+    goto error;
+
   mysql->info_buffer= my_malloc(MYSQL_ERRMSG_SIZE, MYF(0));
   mysql->thd= create_embedded_thd(client_flag);
 
diff --git a/sql-common/client.c b/sql-common/client.c
index 9972ca741f2..3247fd8e339 100644
--- a/sql-common/client.c
+++ b/sql-common/client.c
@@ -2020,6 +2020,34 @@ error:
 #endif /* HAVE_OPENSSL && !EMBEDDED_LIBRARY */
 
 
+/**
+  Checks if any SSL option is set for libmysqld embedded server.
+
+  @param  mysql   the connection handle
+  @retval 0       success
+  @retval 1       failure
+*/
+#ifdef EMBEDDED_LIBRARY
+int embedded_ssl_check(MYSQL *mysql)
+{
+  if (mysql->options.ssl_key || mysql->options.ssl_cert ||
+      mysql->options.ssl_ca || mysql->options.ssl_capath ||
+      mysql->options.ssl_cipher ||
+      mysql->options.client_flag & CLIENT_SSL_VERIFY_SERVER_CERT ||
+      (mysql->options.extension &&
+        mysql->options.extension->ssl_mode == SSL_MODE_REQUIRED))
+  {
+     set_mysql_extended_error(mysql, CR_SSL_CONNECTION_ERROR, unknown_sqlstate,
+                              ER(CR_SSL_CONNECTION_ERROR),
+                              "Embedded server libmysqld library doesn't support "
+                              "SSL connections");
+     return 1;
+  }
+  return 0;
+}
+#endif
+
+
 /*
   Note that the mysql argument must be initialized with mysql_init()
   before calling mysql_real_connect !
@@ -3592,6 +3620,11 @@ CLI_MYSQL_REAL_CONNECT(MYSQL *mysql,const char *host, const char *user,
 
   mysql->client_flag= client_flag;
 
+#ifdef EMBEDDED_LIBRARY
+  if (embedded_ssl_check(mysql))
+    goto error;
+#endif
+
   /*
     Part 2: invoke the plugin to send the authentication data to the server
   */
@@ -4271,10 +4304,14 @@ mysql_options(MYSQL *mysql,enum mysql_option option, const void *arg)
     mysql->reconnect= *(my_bool *) arg;
     break;
   case MYSQL_OPT_SSL_VERIFY_SERVER_CERT:
+#if defined(HAVE_OPENSSL) && !defined(EMBEDDED_LIBRARY)
     if (*(my_bool*) arg)
       mysql->options.client_flag|= CLIENT_SSL_VERIFY_SERVER_CERT;
     else
       mysql->options.client_flag&= ~CLIENT_SSL_VERIFY_SERVER_CERT;
+#elif defined(EMBEDDED_LIBRARY)
+    DBUG_RETURN(1);
+#endif
     break;
   case MYSQL_PLUGIN_DIR:
     EXTENSION_SET_STRING(&mysql->options, plugin_dir, arg);
@@ -4288,11 +4325,15 @@ mysql_options(MYSQL *mysql,enum mysql_option option, const void *arg)
       (*(my_bool*) arg) ? TRUE : FALSE;
     break;
   case MYSQL_OPT_SSL_MODE:
+#if defined(HAVE_OPENSSL) && !defined(EMBEDDED_LIBRARY)
     if (*(uint *) arg == SSL_MODE_REQUIRED)
     {
       ENSURE_EXTENSIONS_PRESENT(&mysql->options);
       mysql->options.extension->ssl_mode= SSL_MODE_REQUIRED;
     }
+#elif defined(EMBEDDED_LIBRARY)
+    DBUG_RETURN(1);
+#endif
     break;
   default:
     DBUG_RETURN(1);

From bd5ca6acece65858591c2bf54f86ff34aeea2821 Mon Sep 17 00:00:00 2001
From: Arun Kuruvila <arun.kuruvila@oracle.com>
Date: Mon, 21 May 2018 08:42:45 +0530
Subject: [PATCH 008/164] Bug#25541037: MYSQL BUG ON DELETE

Description:- MyISAM table gets corrupted with concurrent
executions of INSERT, DELETE statements in a particular
sequence.

Analysis:- Due to the inappropriate manipulation of w_lock
and r_lock associated with a MyISAM table, there arises a
scenario where the table's state information becomes
invalid.

Fix:- A lock is introduced to resolve this issue.
---
 storage/myisam/ha_myisam.cc | 6 +++++-
 storage/myisam/mi_check.c   | 6 +++++-
 storage/myisam/mi_locking.c | 6 +++++-
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc
index 21cbef32188..6eed1be1ba9 100644
--- a/storage/myisam/ha_myisam.cc
+++ b/storage/myisam/ha_myisam.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -1172,10 +1172,14 @@ int ha_myisam::repair(THD *thd, MI_CHECK &param, bool do_optimize)
     if (file->s->base.auto_key)
       update_auto_increment_key(&param, file, 1);
     if (optimize_done)
+    {
+      mysql_mutex_lock(&share->intern_lock);
       error = update_state_info(&param, file,
 				UPDATE_TIME | UPDATE_OPEN_COUNT |
 				(local_testflag &
 				 T_STATISTICS ? UPDATE_STAT : 0));
+      mysql_mutex_unlock(&share->intern_lock);
+    }
     info(HA_STATUS_NO_LOCK | HA_STATUS_TIME | HA_STATUS_VARIABLE |
 	 HA_STATUS_CONST);
     if (rows != file->state->records && ! (param.testflag & T_VERY_SILENT))
diff --git a/storage/myisam/mi_check.c b/storage/myisam/mi_check.c
index fe0d4c9c30b..7134cfc265a 100644
--- a/storage/myisam/mi_check.c
+++ b/storage/myisam/mi_check.c
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -4474,6 +4474,10 @@ int update_state_info(MI_CHECK *param, MI_INFO *info,uint update)
     int error;
     uint r_locks=share->r_locks,w_locks=share->w_locks;
     share->r_locks= share->w_locks= share->tot_locks= 0;
+
+    DBUG_EXECUTE_IF("simulate_incorrect_share_wlock_value",
+                    DEBUG_SYNC_C("after_share_wlock_set_to_0"););
+
     error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK);
     share->r_locks=r_locks;
     share->w_locks=w_locks;
diff --git a/storage/myisam/mi_locking.c b/storage/myisam/mi_locking.c
index 4a33e838fb9..dcfeaca5b8c 100644
--- a/storage/myisam/mi_locking.c
+++ b/storage/myisam/mi_locking.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2000, 2017, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -229,6 +229,10 @@ int mi_lock_database(MI_INFO *info, int lock_type)
       info->invalidator=info->s->invalidator;
       share->w_locks++;
       share->tot_locks++;
+
+      DBUG_EXECUTE_IF("simulate_incorrect_share_wlock_value",
+                      DEBUG_SYNC_C("after_share_wlock_increment"););
+
       info->s->in_use= list_add(info->s->in_use, &info->in_use);
       break;
     default:

From eee5bd9a2e2949ef468670f1ec4431a4ce54ada0 Mon Sep 17 00:00:00 2001
From: Faustin Lammler <faustin@falared.net>
Date: Tue, 3 Apr 2018 10:33:11 -0300
Subject: [PATCH 009/164] Package dependency case problem

dummy commit, add carriage return

rollback and add travis build

remove travis CI config
---
 debian/control | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/control b/debian/control
index 862cd4f3cea..79ccf8c8fd4 100644
--- a/debian/control
+++ b/debian/control
@@ -373,10 +373,10 @@ Description: MariaDB database regression test suite (metapackage depending on th
 Package: mariadb-connect-engine-10.0
 Section: database
 Architecture: any
-Depends: libxml2, mariadb-server-10.0|mariadb-galera-server-10.0, unixODBC
+Depends: libxml2, mariadb-server-10.0|mariadb-galera-server-10.0, unixodbc
 Build-depends: libxml2-dev,
                mariadb-server-10.0|mariadb-galera-server-10.0,
-               unixODBC-dev
+               unixodbc-dev
 Description: Connect storage engine for MariaDB
  Connect engine supports a number of file formats (dbf, xml, txt, bin, etc),
  connections to ODBC tables and remote MySQL tables, as well as a number of

From 15c6d6a94a20053ded2e115d21e7d285b583c9ef Mon Sep 17 00:00:00 2001
From: sjaakola <seppo.jaakola@iki.fi>
Date: Thu, 7 Jun 2018 17:12:38 +0300
Subject: [PATCH 010/164] MDEV-14612 wsrep_sst_mariabackup unnecessarily
 converts address to host name

SST script for mariabackup (wsrep_sst_mariabackup.sh) does reverse DNS to convert
the joiner's IP address to domain name, and this is not always a wanted feature.

It is not understood why this remote host DNS conversion is in the script,
in the first place, but conversion was originally added by commit:

commit 3d8aacba861044843417b710fff208b8bd6a53c4
Author: Nirbhay Choubey <nirbhay.choubey@gmail.com>
Date:   Wed Feb 22 15:58:45 2017 -0500

In this commit, REMOTEHOST variable is removed and replaced by REMOTEIP. All
references to joiner is therefore by IP address only.
---
 scripts/wsrep_sst_mariabackup.sh | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh
index 549161aa8f3..8bc5954e45a 100644
--- a/scripts/wsrep_sst_mariabackup.sh
+++ b/scripts/wsrep_sst_mariabackup.sh
@@ -33,7 +33,6 @@ ssystag=""
 XTRABACKUP_PID=""
 SST_PORT=""
 REMOTEIP=""
-REMOTEHOST=""
 tcert=""
 tpem=""
 tkey=""
@@ -225,7 +224,7 @@ get_transfer()
                 tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tpem},cafile=${tcert}${sockopt} stdio"
             else
                 wsrep_log_info "Encrypting with cert=${tpem}, cafile=${tcert}"
-                tcmd="socat -u stdio openssl-connect:${REMOTEHOST}:${TSST_PORT},cert=${tpem},cafile=${tcert}${sockopt}"
+                tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${tpem},cafile=${tcert}${sockopt}"
             fi
         elif [[ $encrypt -eq 3 ]];then
             wsrep_log_info "Using openssl based encryption with socat: with key and crt"
@@ -248,7 +247,7 @@ get_transfer()
                     tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${tpem},key=${tkey},verify=0${sockopt}"
                 else
                     wsrep_log_info "Encrypting with cert=${tpem}, key=${tkey}, cafile=${tcert}"
-                    tcmd="socat -u stdio openssl-connect:${REMOTEHOST}:${TSST_PORT},cert=${tpem},key=${tkey},cafile=${tcert}${sockopt}"
+                    tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${tpem},key=${tkey},cafile=${tcert}${sockopt}"
                 fi
             fi
 
@@ -518,10 +517,6 @@ setup_ports()
     if [[ "$WSREP_SST_OPT_ROLE"  == "donor" ]];then
         SST_PORT=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $2 }')
         REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F ':' '{ print $1 }')
-        REMOTEHOST=$(getent hosts $REMOTEIP | awk '{ print $2 }')
-        if [[ -z $REMOTEHOST ]];then
-            REMOTEHOST=$REMOTEIP
-        fi
         lsn=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $4 }')
         sst_ver=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $5 }')
     else

From e48d775c6f066add457fa8cfb2ebc4d5ff0c7613 Mon Sep 17 00:00:00 2001
From: Ivo Roylev <ivo.roylev@oracle.com>
Date: Thu, 14 Jun 2018 17:27:54 +0300
Subject: [PATCH 011/164] Bug#27980823: HEAP OVERFLOW VULNERABILITIES IN MYSQL
 CLIENT LIBRARY

(cherry picked from commit b5b986b2cbd9a7848dc3f48e5c42b6d4e1e5fb22)
---
 sql-common/client.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/sql-common/client.c b/sql-common/client.c
index 3247fd8e339..7938403db59 100644
--- a/sql-common/client.c
+++ b/sql-common/client.c
@@ -1505,7 +1505,8 @@ unpack_fields(MYSQL *mysql, MYSQL_DATA *data,MEM_ROOT *alloc,uint fields,
     {
       uchar *pos;
       /* fields count may be wrong */
-      DBUG_ASSERT((uint) (field - result) < fields);
+      if (field < result || (uint) (field - result) >= fields)
+        DBUG_RETURN(NULL);
       cli_fetch_lengths(&lengths[0], row->data, default_value ? 8 : 7);
       field->catalog=   strmake_root(alloc,(char*) row->data[0], lengths[0]);
       field->db=        strmake_root(alloc,(char*) row->data[1], lengths[1]);
@@ -1612,6 +1613,7 @@ MYSQL_DATA *cli_read_rows(MYSQL *mysql,MYSQL_FIELD *mysql_fields,
 
   if ((pkt_len= cli_safe_read(mysql)) == packet_error)
     DBUG_RETURN(0);
+  if (pkt_len == 0) DBUG_RETURN(0);
   if (!(result=(MYSQL_DATA*) my_malloc(sizeof(MYSQL_DATA),
 				       MYF(MY_WME | MY_ZEROFILL))))
   {

From e5c6580178a8d660c03fe9fab09c539cedf6fb34 Mon Sep 17 00:00:00 2001
From: mkaruza <mario.karuza@gmail.com>
Date: Tue, 19 Jun 2018 16:09:31 +0200
Subject: [PATCH 012/164] MDEV-15442 xtrabackup-v2 SST donor stuck in
 DONOR/DESYNCED state when joiner is killed

Monitor mysqld process and if it is terminated during sst stream kill xtrabackup-v2 script and children
---
 scripts/wsrep_sst_xtrabackup-v2.sh | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh
index 00efdaeebcf..99bbabbe9be 100644
--- a/scripts/wsrep_sst_xtrabackup-v2.sh
+++ b/scripts/wsrep_sst_xtrabackup-v2.sh
@@ -801,6 +801,27 @@ check_for_version()
     fi
 }
 
+monitor_process()
+{
+    local sst_stream_pid=$1
+
+    while true ; do
+
+        if ! ps --pid "${WSREP_SST_OPT_PARENT}" &>/dev/null; then
+            wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." 
+            kill -- -"${WSREP_SST_OPT_PARENT}"
+            exit 32
+        fi
+
+        if ! ps --pid "${sst_stream_pid}" &>/dev/null; then 
+            break
+        fi
+
+        sleep 0.1
+
+    done
+}
+
 
 if [[ ! -x `which $INNOBACKUPEX_BIN` ]];then 
     wsrep_log_error "innobackupex not in path: $PATH"
@@ -1102,7 +1123,7 @@ then
 
         MAGIC_FILE="${DATA}/${INFO_FILE}"
         wsrep_log_info "Waiting for SST streaming to complete!"
-        wait $jpid
+        monitor_process $jpid
 
         get_proc
 

From 7a7a61998c60b8114994b29d5f649637be8139e0 Mon Sep 17 00:00:00 2001
From: mkaruza <mario.karuza@gmail.com>
Date: Wed, 20 Jun 2018 13:16:34 +0200
Subject: [PATCH 013/164] MDEV-15442 xtrabackup-v2 SST donor stuck in
 DONOR/DESYNCED state when joiner is killed

SST transfer scripts rsync and maribackup now monitor parent process and will cleanup if process dies
---
 scripts/wsrep_sst_mariabackup.sh | 23 ++++++++++++++++++++++-
 scripts/wsrep_sst_rsync.sh       |  2 ++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh
index 549161aa8f3..ea0a6fc0d51 100644
--- a/scripts/wsrep_sst_mariabackup.sh
+++ b/scripts/wsrep_sst_mariabackup.sh
@@ -644,6 +644,27 @@ send_donor()
 
 }
 
+monitor_process()
+{
+    local sst_stream_pid=$1
+
+    while true ; do
+
+        if ! ps --pid "${WSREP_SST_OPT_PARENT}" &>/dev/null; then
+            wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." 
+            kill -- -"${WSREP_SST_OPT_PARENT}"
+            exit 32
+        fi
+
+        if ! ps --pid "${sst_stream_pid}" &>/dev/null; then 
+            break
+        fi
+
+        sleep 0.1
+
+    done
+}
+
 if [[ ! -x `which $INNOBACKUPEX_BIN` ]];then 
     wsrep_log_error "${INNOBACKUPEX_BIN} not in path: $PATH"
     exit 2
@@ -932,7 +953,7 @@ then
 
         MAGIC_FILE="${DATA}/${INFO_FILE}"
         wsrep_log_info "Waiting for SST streaming to complete!"
-        wait $jpid
+        monitor_process $jpid
 
         get_proc
 
diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh
index 3d76d1780e2..81120b66d5a 100644
--- a/scripts/wsrep_sst_rsync.sh
+++ b/scripts/wsrep_sst_rsync.sh
@@ -329,6 +329,8 @@ EOF
     then
         wsrep_log_error \
         "Parent mysqld process (PID:$MYSQLD_PID) terminated unexpectedly."
+        kill -- -"${MYSQLD_PID}"
+        sleep 1
         exit 32
     fi
 

From 90e608c6acd150e727cb0002f36767d0866aa633 Mon Sep 17 00:00:00 2001
From: Alexander Barkov <bar@mariadb.com>
Date: Tue, 26 Jun 2018 11:42:02 +0400
Subject: [PATCH 014/164] A test cleanup for MDEV-15834: fixing failure in "mtr
 --embedded"

---
 mysql-test/suite/vcol/t/vcol_misc.test | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mysql-test/suite/vcol/t/vcol_misc.test b/mysql-test/suite/vcol/t/vcol_misc.test
index a123d21d574..fab705eaaa6 100644
--- a/mysql-test/suite/vcol/t/vcol_misc.test
+++ b/mysql-test/suite/vcol/t/vcol_misc.test
@@ -327,8 +327,10 @@ SET sql_mode=DEFAULT;
 
 --copy_file std_data/frm/t1.frm $MYSQLD_DATADIR/test/t1.frm
 SHOW TABLES;
+--replace_result $MYSQLD_DATADIR ./
 --error ER_NOT_FORM_FILE
 SHOW CREATE TABLE t1;
+--replace_result $MYSQLD_DATADIR ./
 --error ER_NOT_FORM_FILE
 ALTER TABLE t1;
 --remove_file $MYSQLD_DATADIR/test/t1.frm

From 1f6a89a8fdf88029518584f3e906d0e55bc1889a Mon Sep 17 00:00:00 2001
From: Michael Widenius <monty@mariadb.org>
Date: Wed, 27 Jun 2018 13:18:03 +0300
Subject: [PATCH 015/164] Added valgrind suppression for OpenSuse 42.2

---
 mysql-test/valgrind.supp | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/mysql-test/valgrind.supp b/mysql-test/valgrind.supp
index 71e6836365e..c9c06c2634a 100644
--- a/mysql-test/valgrind.supp
+++ b/mysql-test/valgrind.supp
@@ -608,6 +608,19 @@
    fun:dlopen*
 }
 
+#
+# Warning caused by small memory leak in _dl_init
+#
+
+{
+   dl_init memory leak
+   Memcheck:Leak
+   fun:malloc
+   obj:*/libstdc++.so*
+   fun:call_init.part*
+   fun:_dl_init
+}
+
 #
 # In glibc (checked version 2.7), inet_ntoa allocates an 18-byte
 # per-thread static buffer for the return value. That memory is freed

From faef2e6a44de8e1c1301104671d08dfc1deef1e0 Mon Sep 17 00:00:00 2001
From: Michael Widenius <monty@mariadb.org>
Date: Wed, 27 Jun 2018 13:18:30 +0300
Subject: [PATCH 016/164] Added more help text in case mysql_install_db fails.

---
 scripts/mysql_install_db.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh
index 2c5f8d7082d..a8f32526a7e 100644
--- a/scripts/mysql_install_db.sh
+++ b/scripts/mysql_install_db.sh
@@ -211,6 +211,11 @@ cannot_find_file()
   echo "If you compiled from source, you need to run 'make install' to"
   echo "copy the software into the correct location ready for operation."
   echo
+  echo "If you compiled from source, you need to either run 'make install' to"
+  echo "copy the software into the correct location ready for operation."
+  echo "If you don't want to do a full install, you can use the --srcdir"
+  echo "option to only install the mysql database and privilege tables"
+  echo
   echo "If you are using a binary release, you must either be at the top"
   echo "level of the extracted archive, or pass the --basedir option"
   echo "pointing to that location."

From 937c19318819f63d72a66dac5a60c638386512d6 Mon Sep 17 00:00:00 2001
From: Michael Widenius <monty@mariadb.org>
Date: Wed, 27 Jun 2018 13:17:18 +0300
Subject: [PATCH 017/164] Fixed MDEV-16512, crashing on re-execution of failing
 SP

MDEV-16512 Server crashes in find_field_in_table_ref on 2nd
execution of SP referring to non-existing field

Problem was in the natural join code that it changed TABLE_LIST and
Item_fields but didn't restore changed things if things goes wrong
and was not able to re-execute after failure.
Some of the problems could have been avoided if we would have run
fix_fields before doing natural join transformations.

Fixed by marking functions complete AFTER they had executed, instead at
start.
I had also to change some tests that checked if Item_fields are usable.

This doesn't fix all known problems, but at least avoids some crashes.
What should be done in the near future is to mark the statement in the SP
as 'not re-executable' and force a reparse of it on next execution.

Reviewer: Sergei Petrunia <psergey@askmonty.org>
---
 mysql-test/r/join.result | 40 +++++++++++++++++++++++++++++++
 mysql-test/t/join.test   | 52 ++++++++++++++++++++++++++++++++++++++++
 sql/sql_base.cc          | 12 ++++++----
 sql/table.cc             |  6 ++---
 4 files changed, 102 insertions(+), 8 deletions(-)

diff --git a/mysql-test/r/join.result b/mysql-test/r/join.result
index 1e51d55f012..7b0e7807e39 100644
--- a/mysql-test/r/join.result
+++ b/mysql-test/r/join.result
@@ -1497,3 +1497,43 @@ Note	1003	select NULL AS `i1`,`v2`.`i2` AS `i2`,`v2`.`a` AS `a`,`v2`.`b` AS `b`
 DROP VIEW v2;
 DROP TABLE t1,t2;
 SET optimizer_switch=@save_optimizer_switch;
+#
+# MDEV-16512
+# Server crashes in find_field_in_table_ref on 2nd execution of SP referring to
+# non-existing field
+#
+CREATE TABLE t (i INT);
+CREATE PROCEDURE p() SELECT t1.f FROM t AS t1 JOIN t AS t2 USING (f);
+CALL p;
+ERROR 42S22: Unknown column 'f' in 'from clause'
+CALL p;
+ERROR 42S22: Unknown column 'f' in 'from clause'
+FLUSH TABLES;
+CALL p;
+ERROR 42S22: Unknown column 'f' in 'from clause'
+DROP TABLE t;
+CREATE TABLE t (f INT);
+CALL p;
+ERROR 42S22: Unknown column 'f' in 'from clause'
+DROP TABLE t;
+CREATE TABLE t (i INT);
+CALL p;
+ERROR 42S22: Unknown column 'f' in 'from clause'
+DROP PROCEDURE p;
+DROP TABLE t;
+CREATE TABLE t1 (a INT, b INT);
+CREATE TABLE t2 (a INT);
+CREATE TABLE t3 (a INT, c INT);
+CREATE TABLE t4 (a INT, c INT);
+CREATE TABLE t5 (a INT, c INT);
+CREATE PROCEDURE p1() SELECT c FROM t1 JOIN (t2 LEFT JOIN t3 USING (a) LEFT JOIN t4 USING (a)
+LEFT JOIN t5 USING (a)) USING (a);
+CALL p1;
+ERROR 23000: Column 'c' in field list is ambiguous
+CALL p1;
+ERROR 23000: Column 'c' in field list is ambiguous
+DROP PROCEDURE p1;
+DROP TABLE t1,t2,t3,t4,t5;
+#
+# End of MariaDB 5.5 tests
+#
diff --git a/mysql-test/t/join.test b/mysql-test/t/join.test
index 203a7d377a5..feafac57a7e 100644
--- a/mysql-test/t/join.test
+++ b/mysql-test/t/join.test
@@ -1159,3 +1159,55 @@ DROP VIEW v2;
 DROP TABLE t1,t2;
 
 SET optimizer_switch=@save_optimizer_switch;
+
+--echo #
+--echo # MDEV-16512
+--echo # Server crashes in find_field_in_table_ref on 2nd execution of SP referring to
+--echo # non-existing field
+--echo #
+
+CREATE TABLE t (i INT);
+CREATE PROCEDURE p() SELECT t1.f FROM t AS t1 JOIN t AS t2 USING (f);
+--error ER_BAD_FIELD_ERROR
+CALL p;
+--error ER_BAD_FIELD_ERROR
+CALL p;
+FLUSH TABLES;
+--error ER_BAD_FIELD_ERROR
+CALL p;
+DROP TABLE t;
+
+#
+# Fix the table definition to match the using
+#
+
+CREATE TABLE t (f INT);
+#
+# The following shouldn't fail as the table is now matching the using
+#
+--error ER_BAD_FIELD_ERROR
+CALL p;
+DROP TABLE t;
+CREATE TABLE t (i INT);
+--error ER_BAD_FIELD_ERROR
+CALL p;
+DROP PROCEDURE p;
+DROP TABLE t;
+
+CREATE TABLE t1 (a INT, b INT);
+CREATE TABLE t2 (a INT);
+CREATE TABLE t3 (a INT, c INT);
+CREATE TABLE t4 (a INT, c INT);
+CREATE TABLE t5 (a INT, c INT);
+CREATE PROCEDURE p1() SELECT c FROM t1 JOIN (t2 LEFT JOIN t3 USING (a) LEFT JOIN t4 USING (a)
+LEFT JOIN t5 USING (a)) USING (a);
+--error ER_NON_UNIQ_ERROR
+CALL p1;
+--error ER_NON_UNIQ_ERROR
+CALL p1;
+DROP PROCEDURE p1;
+DROP TABLE t1,t2,t3,t4,t5;
+
+--echo #
+--echo # End of MariaDB 5.5 tests
+--echo #
diff --git a/sql/sql_base.cc b/sql/sql_base.cc
index 71bbb538b5c..8ffb7bc118b 100644
--- a/sql/sql_base.cc
+++ b/sql/sql_base.cc
@@ -7687,6 +7687,7 @@ store_natural_using_join_columns(THD *thd, TABLE_LIST *natural_using_join,
   Query_arena *arena, backup;
   bool result= TRUE;
   List<Natural_join_column> *non_join_columns;
+  List<Natural_join_column> *join_columns;
   DBUG_ENTER("store_natural_using_join_columns");
 
   DBUG_ASSERT(!natural_using_join->join_columns);
@@ -7694,7 +7695,7 @@ store_natural_using_join_columns(THD *thd, TABLE_LIST *natural_using_join,
   arena= thd->activate_stmt_arena_if_needed(&backup);
 
   if (!(non_join_columns= new List<Natural_join_column>) ||
-      !(natural_using_join->join_columns= new List<Natural_join_column>))
+      !(join_columns= new List<Natural_join_column>))
     goto err;
 
   /* Append the columns of the first join operand. */
@@ -7703,7 +7704,7 @@ store_natural_using_join_columns(THD *thd, TABLE_LIST *natural_using_join,
     nj_col_1= it_1.get_natural_column_ref();
     if (nj_col_1->is_common)
     {
-      natural_using_join->join_columns->push_back(nj_col_1);
+      join_columns->push_back(nj_col_1);
       /* Reset the common columns for the next call to mark_common_columns. */
       nj_col_1->is_common= FALSE;
     }
@@ -7724,7 +7725,7 @@ store_natural_using_join_columns(THD *thd, TABLE_LIST *natural_using_join,
     {
       const char *using_field_name_ptr= using_field_name->c_ptr();
       List_iterator_fast<Natural_join_column>
-        it(*(natural_using_join->join_columns));
+        it(*join_columns);
       Natural_join_column *common_field;
 
       for (;;)
@@ -7757,7 +7758,8 @@ store_natural_using_join_columns(THD *thd, TABLE_LIST *natural_using_join,
   }
 
   if (non_join_columns->elements > 0)
-    natural_using_join->join_columns->concat(non_join_columns);
+    join_columns->concat(non_join_columns);
+  natural_using_join->join_columns= join_columns;
   natural_using_join->is_join_columns_complete= TRUE;
 
   result= FALSE;
@@ -7989,7 +7991,6 @@ static bool setup_natural_join_row_types(THD *thd,
     DBUG_PRINT("info", ("using cached setup_natural_join_row_types"));
     DBUG_RETURN(false);
   }
-  context->select_lex->first_natural_join_processing= false;
 
   List_iterator_fast<TABLE_LIST> table_ref_it(*from_clause);
   TABLE_LIST *table_ref; /* Current table reference. */
@@ -8034,6 +8035,7 @@ static bool setup_natural_join_row_types(THD *thd,
     change on re-execution
   */
   context->natural_join_first_table= context->first_name_resolution_table;
+  context->select_lex->first_natural_join_processing= false;
   DBUG_RETURN (false);
 }
 
diff --git a/sql/table.cc b/sql/table.cc
index 87a249defa0..48aa445e1aa 100644
--- a/sql/table.cc
+++ b/sql/table.cc
@@ -5487,7 +5487,7 @@ Field_iterator_table_ref::get_or_create_column_ref(THD *thd, TABLE_LIST *parent_
     nj_col= natural_join_it.column_ref();
     DBUG_ASSERT(nj_col);
   }
-  DBUG_ASSERT(!nj_col->table_field ||
+  DBUG_ASSERT(!nj_col->table_field || !nj_col->table_field->field ||
               nj_col->table_ref->table == nj_col->table_field->field->table);
 
   /*
@@ -5536,7 +5536,7 @@ Field_iterator_table_ref::get_or_create_column_ref(THD *thd, TABLE_LIST *parent_
 
   RETURN
     #     Pointer to a column of a natural join (or its operand)
-    NULL  No memory to allocate the column
+    NULL  We didn't originally have memory to allocate the column
 */
 
 Natural_join_column *
@@ -5552,7 +5552,7 @@ Field_iterator_table_ref::get_natural_column_ref()
   */
   nj_col= natural_join_it.column_ref();
   DBUG_ASSERT(nj_col &&
-              (!nj_col->table_field ||
+              (!nj_col->table_field || !nj_col->table_field->field ||
                nj_col->table_ref->table == nj_col->table_field->field->table));
   return nj_col;
 }

From 9d41dd2f39f5a0c840d77e5fb7fc8d1396bf1a33 Mon Sep 17 00:00:00 2001
From: Sergey Vojtovich <svoj@mariadb.org>
Date: Wed, 27 Jun 2018 15:34:11 +0400
Subject: [PATCH 018/164] MDEV-8540 - Crash on server shutdown since 10.0.16

For the purpose of reporting an error to error log, shutdown thread was
attempting to access current_thd->variables.lc_messages->errmsgs->errmsgs.
Whereas current_thd was NULL.

We should log errors according to global lc_messages setting instead of
session setting.
---
 sql/log.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/sql/log.cc b/sql/log.cc
index 5b4ba87ef52..2504b5e2d06 100644
--- a/sql/log.cc
+++ b/sql/log.cc
@@ -2499,14 +2499,14 @@ void MYSQL_LOG::close(uint exiting)
     if (log_type == LOG_BIN && mysql_file_sync(log_file.file, MYF(MY_WME)) && ! write_error)
     {
       write_error= 1;
-      sql_print_error(ER_THD_OR_DEFAULT(current_thd, ER_ERROR_ON_WRITE), name, errno);
+      sql_print_error(ER_DEFAULT(ER_ERROR_ON_WRITE), name, errno);
     }
 
     if (!(exiting & LOG_CLOSE_DELAYED_CLOSE) &&
         mysql_file_close(log_file.file, MYF(MY_WME)) && ! write_error)
     {
       write_error= 1;
-      sql_print_error(ER_THD_OR_DEFAULT(current_thd, ER_ERROR_ON_WRITE), name, errno);
+      sql_print_error(ER_DEFAULT(ER_ERROR_ON_WRITE), name, errno);
     }
   }
 
@@ -2690,7 +2690,7 @@ err:
   if (!write_error)
   {
     write_error= 1;
-    sql_print_error(ER(ER_ERROR_ON_WRITE), name, errno);
+    sql_print_error(ER_DEFAULT(ER_ERROR_ON_WRITE), name, errno);
   }
   mysql_mutex_unlock(&LOCK_log);
   return TRUE;
@@ -2866,7 +2866,7 @@ bool MYSQL_QUERY_LOG::write(THD *thd, time_t current_time,
       if (! write_error)
       {
         write_error= 1;
-        sql_print_error(ER(ER_ERROR_ON_WRITE), name, tmp_errno);
+        sql_print_error(ER_DEFAULT(ER_ERROR_ON_WRITE), name, tmp_errno);
       }
     }
   }
@@ -6385,7 +6385,7 @@ void MYSQL_BIN_LOG::close(uint exiting)
     if (mysql_file_close(index_file.file, MYF(0)) < 0 && ! write_error)
     {
       write_error= 1;
-      sql_print_error(ER(ER_ERROR_ON_WRITE), index_file_name, errno);
+      sql_print_error(ER_DEFAULT(ER_ERROR_ON_WRITE), index_file_name, errno);
     }
   }
   log_state= (exiting & LOG_CLOSE_TO_BE_OPENED) ? LOG_TO_BE_OPENED : LOG_CLOSED;

From f46acd4a3a74c57a31226b11186746bce6c98813 Mon Sep 17 00:00:00 2001
From: Sergei Petrunia <psergey@askmonty.org>
Date: Fri, 29 Jun 2018 14:00:00 +0300
Subject: [PATCH 019/164] Adopt Debian's fix-FTBFS-on-GNU-Hurd.patch.

- Took the original patch by Ondrej Sury;
- Applied a fix for a known problem in the patch:
   https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=882062
- Fixed a few other issues
---
 cmake/dtrace.cmake                            |  4 ++
 cmake/package_name.cmake                      |  3 ++
 extra/mariabackup/backup_copy.cc              | 34 +++++++++-----
 sql/wsrep_binlog.cc                           | 45 ++++++++++++++-----
 storage/mroonga/vendor/groonga/config.h.cmake |  1 +
 5 files changed, 67 insertions(+), 20 deletions(-)

diff --git a/cmake/dtrace.cmake b/cmake/dtrace.cmake
index d7ab0f31991..bb45eaf865b 100644
--- a/cmake/dtrace.cmake
+++ b/cmake/dtrace.cmake
@@ -46,6 +46,10 @@ MACRO(CHECK_DTRACE)
      AND NOT CMAKE_SYSTEM_NAME MATCHES "SunOS")
    SET(ENABLE_DTRACE ON CACHE BOOL "Enable dtrace")
  ENDIF()
+ # On GNU/Hurd, dtrace is not supported
+ IF(DTRACE AND CMAKE_SYSTEM_NAME MATCHES "GNU")
+   SET(ENABLE_DTRACE OFF CACHE BOOL "Disable dtrace")
+ ENDIF()
  SET(HAVE_DTRACE ${ENABLE_DTRACE})
  IF(CMAKE_SYSTEM_NAME MATCHES "SunOS")
    IF(CMAKE_SIZEOF_VOID_P EQUAL 4)
diff --git a/cmake/package_name.cmake b/cmake/package_name.cmake
index 4930a6bf40a..a8bd1596f47 100644
--- a/cmake/package_name.cmake
+++ b/cmake/package_name.cmake
@@ -77,6 +77,9 @@ IF(NOT VERSION)
           SET(DEFAULT_MACHINE "i386")
         ENDIF()
       ENDIF()
+    ELSEIF(CMAKE_SYSTEM_NAME MATCHES "GNU")
+      SET(DEFAULT_PLATFORM "GNU")
+      SET(DEFAULT_MACHINE "i386")
     ELSEIF(CMAKE_SYSTEM_NAME MATCHES "Darwin")
       IF(CMAKE_OSX_DEPLOYMENT_TARGET)
         SET(DEFAULT_PLATFORM "osx${CMAKE_OSX_DEPLOYMENT_TARGET}")
diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc
index d3253348d72..7cdc6b8e3a6 100644
--- a/extra/mariabackup/backup_copy.cc
+++ b/extra/mariabackup/backup_copy.cc
@@ -623,11 +623,14 @@ static
 int
 mkdirp(const char *pathname, int Flags, myf MyFlags)
 {
-	char parent[PATH_MAX], *p;
+	char *parent, *p;
+	int len = strlen(pathname) + 1;
 
 	/* make a parent directory path */
-	strncpy(parent, pathname, sizeof(parent));
-	parent[sizeof(parent) - 1] = 0;
+	if (!(parent= (char *)malloc(len)))
+          return(-1);
+	strncpy(parent, pathname, len);
+	parent[len-1]= 0;
 
 	for (p = parent + strlen(parent);
 	    !is_path_separator(*p) && p != parent; p--);
@@ -636,19 +639,23 @@ mkdirp(const char *pathname, int Flags, myf MyFlags)
 
 	/* try to make parent directory */
 	if (p != parent && mkdirp(parent, Flags, MyFlags) != 0) {
+		free(parent);
 		return(-1);
 	}
 
 	/* make this one if parent has been made */
 	if (my_mkdir(pathname, Flags, MyFlags) == 0) {
+		free(parent);
 		return(0);
 	}
 
 	/* if it already exists that is fine */
 	if (errno == EEXIST) {
+		free(parent);
 		return(0);
 	}
 
+	free(parent);
 	return(-1);
 }
 
@@ -658,17 +665,24 @@ bool
 equal_paths(const char *first, const char *second)
 {
 #ifdef HAVE_REALPATH
-	char real_first[PATH_MAX];
-	char real_second[PATH_MAX];
+	char *real_first, *real_second;
+	int result;
 
-	if (realpath(first, real_first) == NULL) {
-		return false;
-	}
-	if (realpath(second, real_second) == NULL) {
+	real_first = realpath(first, 0);
+	if (real_first == NULL) {
 		return false;
 	}
 
-	return (strcmp(real_first, real_second) == 0);
+	real_second = realpath(second, 0);
+	if (real_second == NULL) {
+		free(real_first);
+		return false;
+	}
+
+	result = strcmp(real_first, real_second);
+	free(real_first);
+	free(real_second);
+	return result == 0;
 #else
 	return strcmp(first, second) == 0;
 #endif
diff --git a/sql/wsrep_binlog.cc b/sql/wsrep_binlog.cc
index 998f4e72157..52934ff10cd 100644
--- a/sql/wsrep_binlog.cc
+++ b/sql/wsrep_binlog.cc
@@ -318,13 +318,23 @@ int wsrep_write_cache(wsrep_t*  const wsrep,
 
 void wsrep_dump_rbr_buf(THD *thd, const void* rbr_buf, size_t buf_len)
 {
-  char filename[PATH_MAX]= {0};
-  int len= snprintf(filename, PATH_MAX, "%s/GRA_%ld_%lld.log",
+  int len= snprintf(NULL, 0, "%s/GRA_%ld_%lld.log",
                     wsrep_data_home_dir, thd->thread_id,
                     (long long)wsrep_thd_trx_seqno(thd));
-  if (len >= PATH_MAX)
+  if (len < 0)
   {
-    WSREP_ERROR("RBR dump path too long: %d, skipping dump.", len);
+    WSREP_ERROR("snprintf error: %d, skipping dump.", len);
+    DBUG_VOID_RETURN;
+  }
+
+  char *filename= (char *)malloc(len++);
+  int len1= snprintf(filename, len, "%s/GRA_%ld_%lld.log",
+                    wsrep_data_home_dir, thd->thread_id,
+                    (long long)wsrep_thd_trx_seqno(thd));
+  if (len >= len1)
+  {
+    WSREP_ERROR("RBR dump path truncated: %d, skipping dump.", len);
+    free(filename);
     return;
   }
 
@@ -343,6 +353,7 @@ void wsrep_dump_rbr_buf(THD *thd, const void* rbr_buf, size_t buf_len)
     WSREP_ERROR("Failed to open file '%s': %d (%s)",
                 filename, errno, strerror(errno));
   }
+  free(filename);
 }
 
 /*
@@ -448,19 +459,32 @@ void wsrep_dump_rbr_buf_with_header(THD *thd, const void *rbr_buf,
 {
   DBUG_ENTER("wsrep_dump_rbr_buf_with_header");
 
-  char filename[PATH_MAX]= {0};
   File file;
   IO_CACHE cache;
   Log_event_writer writer(&cache);
   Format_description_log_event *ev=NULL;
 
-  int len= my_snprintf(filename, PATH_MAX, "%s/GRA_%ld_%lld_v2.log",
-                       wsrep_data_home_dir, thd->thread_id,
-                       (long long) wsrep_thd_trx_seqno(thd));
+  longlong thd_trx_seqno= (long long)wsrep_thd_trx_seqno(thd);
 
-  if (len >= PATH_MAX)
+  int len= my_snprintf(NULL, 0, "%s/GRA_%ld_%lld_v2.log",
+                       wsrep_data_home_dir, thd->thread_id,
+                       thd_trx_seqno);
+
+  char *filename;
+  if (len < 0 || !(filename= (char*)malloc(len++)))
   {
-    WSREP_ERROR("RBR dump path too long: %d, skipping dump.", len);
+    WSREP_ERROR("snprintf error: %d, skipping dump.", len);
+    DBUG_VOID_RETURN;
+  }
+
+  int len1= my_snprintf(filename, len, "%s/GRA_%ld_%lld_v2.log",
+                        wsrep_data_home_dir, thd->thread_id,
+                        thd_trx_seqno);
+
+  if (len >= len1)
+  {
+    WSREP_ERROR("RBR dump path truncated: %d, skipping dump.", len);
+    free(filename);
     DBUG_VOID_RETURN;
   }
 
@@ -501,6 +525,7 @@ cleanup2:
   end_io_cache(&cache);
 
 cleanup1:
+  free(filename);
   mysql_file_close(file, MYF(MY_WME));
 
   if (!thd->wsrep_applier) delete ev;
diff --git a/storage/mroonga/vendor/groonga/config.h.cmake b/storage/mroonga/vendor/groonga/config.h.cmake
index bfd0cbdc012..ec67c5eea23 100644
--- a/storage/mroonga/vendor/groonga/config.h.cmake
+++ b/storage/mroonga/vendor/groonga/config.h.cmake
@@ -107,6 +107,7 @@
 #cmakedefine HAVE_SIGNAL_H
 #cmakedefine HAVE_SYS_MMAN_H
 #cmakedefine HAVE_SYS_PARAM_H
+#cmakedefine HAVE_SYS_POLL_H
 #cmakedefine HAVE_SYS_RESOURCE_H
 #cmakedefine HAVE_SYS_SELECT_H
 #cmakedefine HAVE_SYS_SOCKET_H

From 83bf267e0d386c35497f34c6ad7bd802c1c8de6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= <vicentiu@mariadb.org>
Date: Fri, 29 Jun 2018 12:09:38 +0300
Subject: [PATCH 020/164] Fix Internal Compiler Error GCC-6.3.0

Change the float comparison function to use a negated version when
comparing for equality. This actually produces less code when compiling
with optimizations (O3) on.
---
 storage/mroonga/vendor/groonga/lib/ts/ts_expr_node.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/storage/mroonga/vendor/groonga/lib/ts/ts_expr_node.c b/storage/mroonga/vendor/groonga/lib/ts/ts_expr_node.c
index 44378cfae25..a3d838c327f 100644
--- a/storage/mroonga/vendor/groonga/lib/ts/ts_expr_node.c
+++ b/storage/mroonga/vendor/groonga/lib/ts/ts_expr_node.c
@@ -562,7 +562,7 @@ inline static grn_ts_bool
 grn_ts_op_not_equal_float(grn_ts_float lhs, grn_ts_float rhs)
 {
   /* To suppress warnings, "lhs != rhs" is not used. */
-  return (lhs < rhs) || (lhs > rhs);
+  return !grn_ts_op_equal_float(lhs, rhs);
 }
 
 /* grn_ts_op_not_equal_time() returns lhs != rhs. */

From 36ea82617c1506532e863cb241296acc8b657243 Mon Sep 17 00:00:00 2001
From: Sergei Petrunia <psergey@askmonty.org>
Date: Fri, 29 Jun 2018 18:16:56 +0300
Subject: [PATCH 021/164] Fix a typo a in the commit before the last one

in the "Adopt Debian's fix-FTBFS-on-GNU-Hurd.patch",
DBUG_VOID_RETURN has been used instead of "return"
---
 sql/wsrep_binlog.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/wsrep_binlog.cc b/sql/wsrep_binlog.cc
index 52934ff10cd..f8553d015aa 100644
--- a/sql/wsrep_binlog.cc
+++ b/sql/wsrep_binlog.cc
@@ -324,7 +324,7 @@ void wsrep_dump_rbr_buf(THD *thd, const void* rbr_buf, size_t buf_len)
   if (len < 0)
   {
     WSREP_ERROR("snprintf error: %d, skipping dump.", len);
-    DBUG_VOID_RETURN;
+    return;
   }
 
   char *filename= (char *)malloc(len++);

From 90cb7212742e9ae3a63bd183e171c95bd12d559f Mon Sep 17 00:00:00 2001
From: Igor Babaev <igor@askmonty.org>
Date: Fri, 29 Jun 2018 22:46:38 -0700
Subject: [PATCH 022/164] MDEV-16603 Crash with set join_cache_level=4

When the definition of the index used for hash join was created
in create_hj_key_for_table() it could cause memory overwrite
due to a bug that led to an underestimation of the number of
the index component.
---
 mysql-test/r/join_cache.result | 33 +++++++++++++++++++++++++++++++++
 mysql-test/t/join_cache.test   | 31 +++++++++++++++++++++++++++++++
 sql/sql_select.cc              |  4 ++--
 3 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/mysql-test/r/join_cache.result b/mysql-test/r/join_cache.result
index 386f7119bc8..f1e6fb577c8 100644
--- a/mysql-test/r/join_cache.result
+++ b/mysql-test/r/join_cache.result
@@ -5871,4 +5871,37 @@ SET join_buffer_size = default;
 SET join_buffer_space_limit= default;
 set optimizer_switch=@save_optimizer_switch;
 DROP TABLE t1,t4,t5,t2;
+#
+# MDEV-16603: BNLH for query with materialized semi-join
+#
+set join_cache_level=4;
+CREATE TABLE t1 ( i1 int, v1 varchar(1)) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (7,'x');
+CREATE TABLE t2 (i1 int, v1 varchar(1), KEY v1 (v1,i1)) ENGINE=InnoDB;
+INSERT INTO t2 VALUES
+(NULL,'x'),(1,'x'),(3,'x'),(5,'x'),(8,'x'),(48,'x'),
+(228,'x'),(3,'y'),(1,'z'),(9,'z');
+CREATE TABLE temp
+SELECT t1.i1 AS f1, t1.v1 AS f2 FROM (t2 JOIN t1 ON (t1.v1 = t2.v1));
+SELECT * FROM temp
+WHERE (f1,f2) IN (SELECT t1.i1, t1.v1 FROM (t2 JOIN t1 ON (t1.v1 = t2.v1)));
+f1	f2
+7	x
+7	x
+7	x
+7	x
+7	x
+7	x
+7	x
+EXPLAIN EXTENDED SELECT * FROM temp
+WHERE (f1,f2) IN (SELECT t1.i1, t1.v1 FROM (t2 JOIN t1 ON (t1.v1 = t2.v1)));
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	PRIMARY	<subquery2>	ALL	distinct_key	NULL	NULL	NULL	1	100.00	
+1	PRIMARY	temp	hash_ALL	NULL	#hash#$hj	9	test.t1.i1,test.t1.v1	7	100.00	Using where; Using join buffer (flat, BNLH join)
+2	MATERIALIZED	t1	ALL	NULL	NULL	NULL	NULL	1	100.00	Using where
+2	MATERIALIZED	t2	hash_index	v1	#hash#v1:v1	4:9	test.t1.v1	10	10.00	Using join buffer (flat, BNLH join)
+Warnings:
+Note	1003	select `test`.`temp`.`f1` AS `f1`,`test`.`temp`.`f2` AS `f2` from `test`.`temp` semi join (`test`.`t2` join `test`.`t1`) where ((`test`.`temp`.`f1` = `test`.`t1`.`i1`) and (`test`.`t2`.`v1` = `test`.`t1`.`v1`) and (`test`.`temp`.`f2` = `test`.`t1`.`v1`))
+DROP TABLE t1,t2,temp;
+SET join_cache_level = default;
 set @@optimizer_switch=@save_optimizer_switch;
diff --git a/mysql-test/t/join_cache.test b/mysql-test/t/join_cache.test
index 58a7b885356..d82b4fa6030 100644
--- a/mysql-test/t/join_cache.test
+++ b/mysql-test/t/join_cache.test
@@ -3836,5 +3836,36 @@ set optimizer_switch=@save_optimizer_switch;
 
 DROP TABLE t1,t4,t5,t2;
 
+--echo #
+--echo # MDEV-16603: BNLH for query with materialized semi-join
+--echo #
+
+--source include/have_innodb.inc
+
+set join_cache_level=4;
+
+CREATE TABLE t1 ( i1 int, v1 varchar(1)) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (7,'x');
+
+CREATE TABLE t2 (i1 int, v1 varchar(1), KEY v1 (v1,i1)) ENGINE=InnoDB;
+
+INSERT INTO t2 VALUES
+ (NULL,'x'),(1,'x'),(3,'x'),(5,'x'),(8,'x'),(48,'x'),
+ (228,'x'),(3,'y'),(1,'z'),(9,'z');
+
+CREATE TABLE temp
+SELECT t1.i1 AS f1, t1.v1 AS f2 FROM (t2 JOIN t1 ON (t1.v1 = t2.v1));
+
+let $q =
+SELECT * FROM temp
+WHERE (f1,f2) IN (SELECT t1.i1, t1.v1 FROM (t2 JOIN t1 ON (t1.v1 = t2.v1)));
+
+eval $q;
+eval EXPLAIN EXTENDED $q;
+
+DROP TABLE t1,t2,temp;
+
+SET join_cache_level = default;
+
 # this must be the last command in the file
 set @@optimizer_switch=@save_optimizer_switch;
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index b79431ae2f6..700b7b37928 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -7994,7 +7994,6 @@ static bool create_hj_key_for_table(JOIN *join, JOIN_TAB *join_tab,
       if (first_keyuse)
       {
         key_parts++;
-        first_keyuse= FALSE;
       }
       else
       {
@@ -8004,7 +8003,7 @@ static bool create_hj_key_for_table(JOIN *join, JOIN_TAB *join_tab,
           if (curr->keypart == keyuse->keypart &&
               !(~used_tables & curr->used_tables) &&
               join_tab->keyuse_is_valid_for_access_in_chosen_plan(join,
-                                                                  keyuse) &&
+                                                                  curr) &&
               are_tables_local(join_tab, curr->used_tables))
             break;
         }
@@ -8012,6 +8011,7 @@ static bool create_hj_key_for_table(JOIN *join, JOIN_TAB *join_tab,
            key_parts++;
       }
     }
+    first_keyuse= FALSE;
     keyuse++;
   } while (keyuse->table == table && keyuse->is_for_hash_join());
   if (!key_parts)

From 1d10c9afe0f2f4fba73892e6c12ea6efe90d5931 Mon Sep 17 00:00:00 2001
From: Sergei Petrunia <psergey@askmonty.org>
Date: Mon, 2 Jul 2018 15:29:22 +0300
Subject: [PATCH 023/164] Post-fix to "Adopt Debian's
 fix-FTBFS-on-GNU-Hurd.patch", part #2.

"my_snprintf(NULL, 0, ...)" does not follow what snprintf does. Change
the call in wsrep_dump_rbr_buf_with_header to use snprintf (note that
wsrep_dump_rbr_buf was using snprintf all the way).

Fix an off-by-one error in comparison so it actually prints the output
---
 sql/wsrep_binlog.cc | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/sql/wsrep_binlog.cc b/sql/wsrep_binlog.cc
index f8553d015aa..902190d4772 100644
--- a/sql/wsrep_binlog.cc
+++ b/sql/wsrep_binlog.cc
@@ -327,11 +327,11 @@ void wsrep_dump_rbr_buf(THD *thd, const void* rbr_buf, size_t buf_len)
     return;
   }
 
-  char *filename= (char *)malloc(len++);
+  char *filename= (char *)malloc(len+1);
   int len1= snprintf(filename, len, "%s/GRA_%ld_%lld.log",
                     wsrep_data_home_dir, thd->thread_id,
                     (long long)wsrep_thd_trx_seqno(thd));
-  if (len >= len1)
+  if (len > len1)
   {
     WSREP_ERROR("RBR dump path truncated: %d, skipping dump.", len);
     free(filename);
@@ -466,22 +466,22 @@ void wsrep_dump_rbr_buf_with_header(THD *thd, const void *rbr_buf,
 
   longlong thd_trx_seqno= (long long)wsrep_thd_trx_seqno(thd);
 
-  int len= my_snprintf(NULL, 0, "%s/GRA_%ld_%lld_v2.log",
-                       wsrep_data_home_dir, thd->thread_id,
-                       thd_trx_seqno);
+  int len= snprintf(NULL, 0, "%s/GRA_%ld_%lld_v2.log",
+                    wsrep_data_home_dir, thd->thread_id,
+                    thd_trx_seqno);
 
   char *filename;
-  if (len < 0 || !(filename= (char*)malloc(len++)))
+  if (len < 0 || !(filename= (char*)malloc(len+1)))
   {
     WSREP_ERROR("snprintf error: %d, skipping dump.", len);
     DBUG_VOID_RETURN;
   }
 
-  int len1= my_snprintf(filename, len, "%s/GRA_%ld_%lld_v2.log",
-                        wsrep_data_home_dir, thd->thread_id,
-                        thd_trx_seqno);
+  int len1= snprintf(filename, len, "%s/GRA_%ld_%lld_v2.log",
+                     wsrep_data_home_dir, thd->thread_id,
+                     thd_trx_seqno);
 
-  if (len >= len1)
+  if (len > len1)
   {
     WSREP_ERROR("RBR dump path truncated: %d, skipping dump.", len);
     free(filename);

From 0897a25c0f0b9b86590fd581795aa64b3fbef009 Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Mon, 2 Jul 2018 15:02:31 +0100
Subject: [PATCH 024/164] MDEV-16596 : Windows - redo log does not work on
 native 4K sector disks.

Disks with native 4K sectors need 4K alignment and size for  unbuffered IO
(i.e for files opened with FILE_FLAG_NO_BUFFERING)

Innodb opens redo log with FILE_FLAG_NO_BUFFERING, however it always does
512byte IOs. Thus, the IO on 4K native sectors will fail, rendering
Innodb non-functional.

The fix is to check whether OS_FILE_LOG_BLOCK_SIZE is multiple of logical
sector size, and if it is not, reopen the redo log without
FILE_FLAG_NO_BUFFERING flag.
---
 cmake/os/Windows.cmake       |  2 +-
 storage/xtradb/os/os0file.cc | 37 ++++++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/cmake/os/Windows.cmake b/cmake/os/Windows.cmake
index e4221227d91..b69ae9bebf8 100644
--- a/cmake/os/Windows.cmake
+++ b/cmake/os/Windows.cmake
@@ -51,7 +51,7 @@ IF(CMAKE_C_COMPILER MATCHES "icl")
 ENDIF()
 
 ADD_DEFINITIONS(-D_WINDOWS -D__WIN__ -D_CRT_SECURE_NO_DEPRECATE)
-ADD_DEFINITIONS(-D_WIN32_WINNT=0x0501)
+ADD_DEFINITIONS(-D_WIN32_WINNT=0x0A00)
 # We do not want the windows.h macros min/max
 ADD_DEFINITIONS(-DNOMINMAX)
 # Speed up build process excluding unused header files
diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc
index 9df42470e5a..bf9db50b7f7 100644
--- a/storage/xtradb/os/os0file.cc
+++ b/storage/xtradb/os/os0file.cc
@@ -1691,6 +1691,31 @@ os_file_set_atomic_writes(
 #endif
 }
 
+/** Check that IO of specific size is possible for the file
+opened with FILE_FLAG_NO_BUFFERING.
+
+The requirement is that IO is multiple of the disk sector size.
+
+@param[in]	file      file handle
+@param[in]	io_size   expected io size
+@return true - unbuffered io of requested size is possible, false otherwise.
+
+@note: this function only works correctly with Windows 8 or later,
+(GetFileInformationByHandleEx with FileStorageInfo is only supported there).
+It will return true on earlier Windows version.
+*/
+static bool unbuffered_io_possible(HANDLE file, size_t io_size)
+{
+	FILE_STORAGE_INFO info;
+	if (GetFileInformationByHandleEx(
+		file, FileStorageInfo, &info, sizeof(info))) {
+		ULONG sector_size = info.LogicalBytesPerSector;
+		if (sector_size)
+			return io_size % sector_size == 0;
+	}
+	return true;
+}
+
 /****************************************************************//**
 NOTE! Use the corresponding macro os_file_create(), not directly
 this function!
@@ -1851,6 +1876,18 @@ os_file_create_func(
 			(LPCTSTR) name, access, share_mode, NULL,
 			create_flag, attributes, NULL);
 
+		/* If FILE_FLAG_NO_BUFFERING was set, check if this can work at all,
+		for expected IO sizes. Reopen without the unbuffered flag, if it is won't work*/
+		if ((file.m_file != INVALID_HANDLE_VALUE)
+			&& (attributes & FILE_FLAG_NO_BUFFERING)
+			&& (type == OS_LOG_FILE)
+			&& !unbuffered_io_possible(file.m_file, OS_FILE_LOG_BLOCK_SIZE)) {
+			ut_a(CloseHandle(file.m_file));
+			attributes &= ~FILE_FLAG_NO_BUFFERING;
+			create_flag = OPEN_ALWAYS;
+			continue;
+		}
+
 		if (file.m_file == INVALID_HANDLE_VALUE) {
 			const char*	operation;
 

From 8c5d64dafb3051749f3762cbad88553568abbb16 Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Mon, 2 Jul 2018 15:22:52 +0100
Subject: [PATCH 025/164] Post-fix after MDEV-8540 - do not close stdin on
 Windows.

It is not open.
---
 sql/mysqld.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index d760986a303..75d6cca7fda 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -5149,8 +5149,10 @@ int win_main(int argc, char **argv)
 int mysqld_main(int argc, char **argv)
 #endif
 {
+#ifndef _WIN32
   /* We can't close stdin just now, because it may be booststrap mode. */
   bool please_close_stdin= fcntl(STDIN_FILENO, F_GETFD) >= 0;
+#endif
 
   /*
     Perform basic thread library and malloc initialization,
@@ -5495,9 +5497,12 @@ int mysqld_main(int argc, char **argv)
                          (char*) "" : mysqld_unix_port),
                          mysqld_port,
                          MYSQL_COMPILATION_COMMENT);
+#ifndef _WIN32
   /* Only close stdin if it was open initinally. */
   if (please_close_stdin)
     fclose(stdin);
+#endif
+
 #if defined(_WIN32) && !defined(EMBEDDED_LIBRARY)
   Service.SetRunning();
 #endif

From a2c0376e08d80d7b7dad8713d1df334b2b81eff9 Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Mon, 2 Jul 2018 17:45:19 +0100
Subject: [PATCH 026/164] Fix build on non-Windows, broken by 0897a25c0f0b9b865

---
 storage/xtradb/os/os0file.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc
index bf9db50b7f7..4246bfbff70 100644
--- a/storage/xtradb/os/os0file.cc
+++ b/storage/xtradb/os/os0file.cc
@@ -1691,6 +1691,7 @@ os_file_set_atomic_writes(
 #endif
 }
 
+#ifdef _WIN32
 /** Check that IO of specific size is possible for the file
 opened with FILE_FLAG_NO_BUFFERING.
 
@@ -1715,6 +1716,7 @@ static bool unbuffered_io_possible(HANDLE file, size_t io_size)
 	}
 	return true;
 }
+#endif
 
 /****************************************************************//**
 NOTE! Use the corresponding macro os_file_create(), not directly

From 24a0a74f5d92a861989fcea17a0c62c1ee7b1299 Mon Sep 17 00:00:00 2001
From: Varun Gupta <varunraiko1803@gmail.com>
Date: Tue, 10 Jul 2018 13:54:04 +0530
Subject: [PATCH 027/164] MDEV-16307: Incorrect results when using BNLH join
 instead of BNL join with views

In this issue we are using derived_with_keys optimization and we are using these keys to do a hash join which is incorrect.
We cannot create keys for dervied tables whose keyparts have types are of BLOB or TEXT type. TEXT or BLOB  columns can only be
indexed over a specified length.
---
 mysql-test/r/derived.result | 33 +++++++++++++++++++++++++++++++++
 mysql-test/t/derived.test   | 21 +++++++++++++++++++++
 sql/table.cc                | 13 +++++++++++--
 3 files changed, 65 insertions(+), 2 deletions(-)

diff --git a/mysql-test/r/derived.result b/mysql-test/r/derived.result
index 54c78dc9f6f..5debfaed5e3 100644
--- a/mysql-test/r/derived.result
+++ b/mysql-test/r/derived.result
@@ -1023,6 +1023,7 @@ INSERT INTO t2 VALUES (NULL),(NULL);
 CREATE TABLE t3 (c VARCHAR(1024) CHARACTER SET utf8, d INT) ENGINE=MyISAM;
 CREATE OR REPLACE ALGORITHM=TEMPTABLE VIEW v3 AS SELECT * FROM t3;
 INSERT INTO t3 VALUES ('abc',NULL),('def',4);
+set @save_join_cache_level= @@join_cache_level;
 SET join_cache_level= 8;
 explain
 SELECT * FROM v1, t2, v3 WHERE a = c AND b = d;
@@ -1052,4 +1053,36 @@ i
 drop procedure pr;
 drop view v1;
 drop table t1;
+set @@join_cache_level= @save_join_cache_level;
+#
+# MDEV-16307: Incorrect results when using BNLH join instead of BNL join with views
+#
+CREATE TABLE t1 (c1 text, c2 int);
+INSERT INTO t1 VALUES ('a',1), ('c',3), ('g',7), ('d',4), ('c',3);
+CREATE TABLE t2 (c1 text, c2 int);
+INSERT INTO t2 VALUES ('b',2), ('c',3);
+CREATE ALGORITHM=TEMPTABLE VIEW v1 AS SELECT * FROM t1;
+explain SELECT v1.c1, v1.c2, t2.c1, t2.c2 FROM v1, t2 WHERE v1.c1=t2.c1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	2	
+1	PRIMARY	<derived2>	ALL	NULL	NULL	NULL	NULL	5	Using where; Using join buffer (flat, BNL join)
+2	DERIVED	t1	ALL	NULL	NULL	NULL	NULL	5	
+SELECT v1.c1, v1.c2, t2.c1, t2.c2 FROM v1, t2 WHERE v1.c1=t2.c1;
+c1	c2	c1	c2
+c	3	c	3
+c	3	c	3
+set @save_join_cache_level= @@join_cache_level;
+set @@join_cache_level=4;
+explain SELECT v1.c1, v1.c2, t2.c1, t2.c2 FROM v1, t2 WHERE v1.c1=t2.c1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	2	Using where
+1	PRIMARY	<derived2>	hash_ALL	NULL	#hash#$hj	3	test.t2.c1	5	Using where; Using join buffer (flat, BNLH join)
+2	DERIVED	t1	ALL	NULL	NULL	NULL	NULL	5	
+SELECT v1.c1, v1.c2, t2.c1, t2.c2 FROM v1, t2 WHERE v1.c1=t2.c1;
+c1	c2	c1	c2
+c	3	c	3
+c	3	c	3
+drop table t1,t2;
+drop view v1;
+set @@join_cache_level= @save_join_cache_level;
 # end of 5.5
diff --git a/mysql-test/t/derived.test b/mysql-test/t/derived.test
index c5b792c8d4d..217203422e0 100644
--- a/mysql-test/t/derived.test
+++ b/mysql-test/t/derived.test
@@ -880,6 +880,7 @@ CREATE TABLE t3 (c VARCHAR(1024) CHARACTER SET utf8, d INT) ENGINE=MyISAM;
 CREATE OR REPLACE ALGORITHM=TEMPTABLE VIEW v3 AS SELECT * FROM t3;
 INSERT INTO t3 VALUES ('abc',NULL),('def',4);
 
+set @save_join_cache_level= @@join_cache_level;
 SET join_cache_level= 8;
 explain
 SELECT * FROM v1, t2, v3 WHERE a = c AND b = d;
@@ -902,5 +903,25 @@ call pr(2);
 drop procedure pr;
 drop view v1;
 drop table t1;
+set @@join_cache_level= @save_join_cache_level;
 
+--echo #
+--echo # MDEV-16307: Incorrect results when using BNLH join instead of BNL join with views
+--echo #
+
+CREATE TABLE t1 (c1 text, c2 int);
+INSERT INTO t1 VALUES ('a',1), ('c',3), ('g',7), ('d',4), ('c',3);
+CREATE TABLE t2 (c1 text, c2 int);
+INSERT INTO t2 VALUES ('b',2), ('c',3);
+CREATE ALGORITHM=TEMPTABLE VIEW v1 AS SELECT * FROM t1;
+
+explain SELECT v1.c1, v1.c2, t2.c1, t2.c2 FROM v1, t2 WHERE v1.c1=t2.c1;
+SELECT v1.c1, v1.c2, t2.c1, t2.c2 FROM v1, t2 WHERE v1.c1=t2.c1;
+set @save_join_cache_level= @@join_cache_level;
+set @@join_cache_level=4;
+explain SELECT v1.c1, v1.c2, t2.c1, t2.c2 FROM v1, t2 WHERE v1.c1=t2.c1;
+SELECT v1.c1, v1.c2, t2.c1, t2.c2 FROM v1, t2 WHERE v1.c1=t2.c1;
+drop table t1,t2;
+drop view v1;
+set @@join_cache_level= @save_join_cache_level;
 --echo # end of 5.5
diff --git a/sql/table.cc b/sql/table.cc
index 48aa445e1aa..f6152a36eef 100644
--- a/sql/table.cc
+++ b/sql/table.cc
@@ -6060,6 +6060,14 @@ void TABLE::create_key_part_by_field(KEY_PART_INFO *key_part_info,
   The function checks whether a possible key satisfies the constraints
   imposed on the keys of any temporary table.
 
+  We need to filter out BLOB columns here, because ref access optimizer creates
+  KEYUSE objects for equalities for non-key columns for two puproses:
+  1. To discover possible keys for derived_with_keys optimization
+  2. To do hash joins
+  For the purpose of #1, KEYUSE objects are not created for "blob_column=..." .
+  However, they might be created for #2. In order to catch that case, we filter
+  them out here.
+
   @return TRUE if the key is valid
   @return FALSE otherwise
 */
@@ -6075,11 +6083,12 @@ bool TABLE::check_tmp_key(uint key, uint key_parts,
   {
     uint fld_idx= next_field_no(arg);
     reg_field= field + fld_idx;
+    if ((*reg_field)->type() == MYSQL_TYPE_BLOB)
+      return FALSE;
     uint fld_store_len= (uint16) (*reg_field)->key_length();
     if ((*reg_field)->real_maybe_null())
       fld_store_len+= HA_KEY_NULL_LENGTH;
-    if ((*reg_field)->type() == MYSQL_TYPE_BLOB ||
-        (*reg_field)->real_type() == MYSQL_TYPE_VARCHAR ||
+    if ((*reg_field)->real_type() == MYSQL_TYPE_VARCHAR ||
         (*reg_field)->type() == MYSQL_TYPE_GEOMETRY)
       fld_store_len+= HA_KEY_BLOB_LENGTH;
     key_len+= fld_store_len;

From 2fbf2277ffec86d69f793534da7043b6dd540780 Mon Sep 17 00:00:00 2001
From: Sergei Petrunia <psergey@askmonty.org>
Date: Wed, 11 Jul 2018 10:43:38 +0300
Subject: [PATCH 028/164] MDEV-15982: Incorrect results when subquery is
 materialized

fix_semijoin_strategies_for_picked_join_order() should set
join->sjm_lookup_tables to be a bitmap of tables inside
SJ-Materialization-Lookup nests.
---
 mysql-test/r/subselect_sj2_mat.result | 51 +++++++++++++++++++++++++++
 mysql-test/t/subselect_sj2_mat.test   | 49 +++++++++++++++++++++++++
 sql/opt_subselect.cc                  |  3 +-
 3 files changed, 102 insertions(+), 1 deletion(-)

diff --git a/mysql-test/r/subselect_sj2_mat.result b/mysql-test/r/subselect_sj2_mat.result
index 19aa3bd3f02..7f97e1aabee 100644
--- a/mysql-test/r/subselect_sj2_mat.result
+++ b/mysql-test/r/subselect_sj2_mat.result
@@ -1658,3 +1658,54 @@ id
 12
 13
 drop table t1;
+#
+# MDEV-15982: Incorrect results when subquery is materialized
+#
+CREATE TABLE `t1` (`id` int(32)  NOT NULL  primary key);
+INSERT INTO `t1` VALUES
+(45), (46), (47), (48), (49), (50), (51), (52), (53), (54), (55), (56), (57), (58), (59), (60), (61), (62),
+(63), (64), (65), (66), (67), (68), (69), (70), (71), (72), (73), (74), (75), (76), (77), (78), (79), (80),
+(81), (82), (83), (84), (85), (86), (87), (88), (89), (90), (91), (92),(93),(94),(95),(96), (97), (98),
+(99), (100), (101), (102), (103), (104), (105), (106), (107), (108), (109), (110), (111), (112), (113),
+(114), (115), (116), (117), (118), (119), (120), (121), (122), (123), (124), (125), (126), (127), (128),
+(129), (130), (131), (132), (133), (134), (135), (136), (137), (138), (139), (140), (141), (142), (143), (144), (145), (146),
+(147), (148), (149), (150), (151), (152), (153), (154), (155), (156), (157), (158), (159), (160), (161),
+(162), (163), (164), (165), (166), (167), (168), (169), (170), (171), (172), (173),
+(174), (175), (176), (177), (178), (179), (180), (181), (182), (183), (2), (3), (4), (5), (6), (19), (35),
+(7), (20), (8), (36), (219), (22), (10), (23), (37), (11), (24);
+CREATE TABLE `t2` (`type` int , `id` int(32) NOT NULL  primary key);
+INSERT INTO `t2` VALUES
+(2,2),(2,3),(1,4),(2,5),(1,6),(1,19),(5,7),(1,20),(1,8),(1,21),(1,9),
+(1,22),(2,10),(1,23),(2,11),(1,24),(1,12),(1,25),(2,13),(2,26),(2,14),
+(2,27),(1,15),(1,28),(3,16),(1,29),(2,17),(1,30),(5,18),(2,1);
+CREATE TABLE `t3` (`ref_id`  int(32) unsigned ,`type` varchar(80),`id` int(32) NOT NULL );
+INSERT INTO `t3` VALUES
+(1,'incident',31),(2,'faux pas',32),
+(5,'oopsies',33),(3,'deniable',34),
+(11,'wasntme',35),(10,'wasntme',36),
+(17,'faux pas',37),(13,'unlikely',38),
+(13,'improbable',39),(14,'incident',40),
+(26,'problem',41),(14,'problem',42),
+(26,'incident',43),(27,'incident',44);
+explain
+SELECT t2.id FROM t2,t1
+WHERE  t2.id IN (SELECT t3.ref_id FROM t3,t1 where t3.id = t1.id) and t2.id = t1.id;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	index	PRIMARY	PRIMARY	4	NULL	30	Using index
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	4	func	1	Using where
+1	PRIMARY	t1	eq_ref	PRIMARY	PRIMARY	4	test.t2.id	1	Using index
+2	MATERIALIZED	t3	ALL	NULL	NULL	NULL	NULL	14	
+2	MATERIALIZED	t1	eq_ref	PRIMARY	PRIMARY	4	test.t3.id	1	Using index
+SELECT t2.id FROM t2,t1
+WHERE  t2.id IN (SELECT t3.ref_id FROM t3,t1 where t3.id = t1.id) and t2.id = t1.id;
+id
+10
+11
+set optimizer_switch='materialization=off';
+SELECT t2.id FROM t2,t1
+WHERE  t2.id IN (SELECT t3.ref_id FROM t3,t1 where t3.id = t1.id) and t2.id = t1.id;
+id
+11
+10
+set optimizer_switch='materialization=on';
+DROP TABLE t1,t2,t3;
diff --git a/mysql-test/t/subselect_sj2_mat.test b/mysql-test/t/subselect_sj2_mat.test
index 0665cdf68fe..43b63459928 100644
--- a/mysql-test/t/subselect_sj2_mat.test
+++ b/mysql-test/t/subselect_sj2_mat.test
@@ -345,3 +345,52 @@ WHERE (
         (t.id IN (0,4,12,13,1,10,3,11))
 );
 drop table t1;
+--echo #
+--echo # MDEV-15982: Incorrect results when subquery is materialized
+--echo #
+
+CREATE TABLE `t1` (`id` int(32)  NOT NULL  primary key);
+INSERT INTO `t1` VALUES
+(45), (46), (47), (48), (49), (50), (51), (52), (53), (54), (55), (56), (57), (58), (59), (60), (61), (62),
+(63), (64), (65), (66), (67), (68), (69), (70), (71), (72), (73), (74), (75), (76), (77), (78), (79), (80),
+(81), (82), (83), (84), (85), (86), (87), (88), (89), (90), (91), (92),(93),(94),(95),(96), (97), (98),
+(99), (100), (101), (102), (103), (104), (105), (106), (107), (108), (109), (110), (111), (112), (113),
+(114), (115), (116), (117), (118), (119), (120), (121), (122), (123), (124), (125), (126), (127), (128),
+(129), (130), (131), (132), (133), (134), (135), (136), (137), (138), (139), (140), (141), (142), (143), (144), (145), (146),
+(147), (148), (149), (150), (151), (152), (153), (154), (155), (156), (157), (158), (159), (160), (161),
+(162), (163), (164), (165), (166), (167), (168), (169), (170), (171), (172), (173),
+(174), (175), (176), (177), (178), (179), (180), (181), (182), (183), (2), (3), (4), (5), (6), (19), (35),
+(7), (20), (8), (36), (219), (22), (10), (23), (37), (11), (24);
+
+CREATE TABLE `t2` (`type` int , `id` int(32) NOT NULL  primary key);
+INSERT INTO `t2` VALUES
+(2,2),(2,3),(1,4),(2,5),(1,6),(1,19),(5,7),(1,20),(1,8),(1,21),(1,9),
+(1,22),(2,10),(1,23),(2,11),(1,24),(1,12),(1,25),(2,13),(2,26),(2,14),
+(2,27),(1,15),(1,28),(3,16),(1,29),(2,17),(1,30),(5,18),(2,1);
+
+CREATE TABLE `t3` (`ref_id`  int(32) unsigned ,`type` varchar(80),`id` int(32) NOT NULL );
+INSERT INTO `t3` VALUES
+(1,'incident',31),(2,'faux pas',32),
+(5,'oopsies',33),(3,'deniable',34),
+(11,'wasntme',35),(10,'wasntme',36),
+(17,'faux pas',37),(13,'unlikely',38),
+(13,'improbable',39),(14,'incident',40),
+(26,'problem',41),(14,'problem',42),
+(26,'incident',43),(27,'incident',44);
+
+explain
+SELECT t2.id FROM t2,t1
+  WHERE  t2.id IN (SELECT t3.ref_id FROM t3,t1 where t3.id = t1.id) and t2.id = t1.id;
+
+SELECT t2.id FROM t2,t1
+  WHERE  t2.id IN (SELECT t3.ref_id FROM t3,t1 where t3.id = t1.id) and t2.id = t1.id;
+
+set optimizer_switch='materialization=off';
+
+SELECT t2.id FROM t2,t1
+  WHERE  t2.id IN (SELECT t3.ref_id FROM t3,t1 where t3.id = t1.id) and t2.id = t1.id;
+set optimizer_switch='materialization=on';
+
+DROP TABLE t1,t2,t3;
+
+
diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc
index ec7b10f20c8..14b2aaee591 100644
--- a/sql/opt_subselect.cc
+++ b/sql/opt_subselect.cc
@@ -3532,7 +3532,8 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join)
       first= tablenr - sjm->tables + 1;
       join->best_positions[first].n_sj_tables= sjm->tables;
       join->best_positions[first].sj_strategy= SJ_OPT_MATERIALIZE;
-      join->sjm_lookup_tables|= s->table->map;
+      for (uint i= first; i < first+ sjm->tables; i++)
+        join->sjm_lookup_tables |= join->best_positions[i].table->table->map;
     }
     else if (pos->sj_strategy == SJ_OPT_MATERIALIZE_SCAN)
     {

From ad9d1e8c3f5a8e1b3e222921e825247aa47c4d23 Mon Sep 17 00:00:00 2001
From: Varun Gupta <varunraiko1803@gmail.com>
Date: Wed, 11 Jul 2018 02:28:42 +0530
Subject: [PATCH 029/164] MDEV-16552: [10.0] ASAN global-buffer-overflow in
 is_stat_table / statistics_for_tables_is_needed

Backport the fix f214d365121 to 10.0

   Author: Sergei Golubchik <serg@mariadb.org>
   Date:   Tue Apr 17 00:44:34 2018 +0200

    ASAN error in is_stat_table()

    don't memcmp beyond the first argument's end

    Also: use my_strcasecmp(table_alias_charset), like elsewhere, not memcmp
---
 mysql-test/r/stat_tables.result        | 7 +++++++
 mysql-test/r/stat_tables_innodb.result | 7 +++++++
 mysql-test/t/stat_tables.test          | 6 ++++++
 sql/sql_statistics.cc                  | 4 ++--
 4 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/mysql-test/r/stat_tables.result b/mysql-test/r/stat_tables.result
index fcced761283..4a608089d7d 100644
--- a/mysql-test/r/stat_tables.result
+++ b/mysql-test/r/stat_tables.result
@@ -516,4 +516,11 @@ use test;
 drop database db1;
 drop database db2;
 drop table t1;
+#
+# MDEV-16552: [10.0] ASAN global-buffer-overflow in is_stat_table / statistics_for_tables_is_needed
+#
+SET use_stat_tables = PREFERABLY;
+SELECT CONVERT_TZ( '1991-09-20 10:11:02', '+00:00', 'GMT' );
+CONVERT_TZ( '1991-09-20 10:11:02', '+00:00', 'GMT' )
+NULL
 set use_stat_tables=@save_use_stat_tables;
diff --git a/mysql-test/r/stat_tables_innodb.result b/mysql-test/r/stat_tables_innodb.result
index 0e866755532..87049b228e5 100644
--- a/mysql-test/r/stat_tables_innodb.result
+++ b/mysql-test/r/stat_tables_innodb.result
@@ -543,6 +543,13 @@ use test;
 drop database db1;
 drop database db2;
 drop table t1;
+#
+# MDEV-16552: [10.0] ASAN global-buffer-overflow in is_stat_table / statistics_for_tables_is_needed
+#
+SET use_stat_tables = PREFERABLY;
+SELECT CONVERT_TZ( '1991-09-20 10:11:02', '+00:00', 'GMT' );
+CONVERT_TZ( '1991-09-20 10:11:02', '+00:00', 'GMT' )
+NULL
 set use_stat_tables=@save_use_stat_tables;
 set optimizer_switch=@save_optimizer_switch_for_stat_tables_test;
 SET SESSION STORAGE_ENGINE=DEFAULT;
diff --git a/mysql-test/t/stat_tables.test b/mysql-test/t/stat_tables.test
index 4cbaa9e27c8..9f94cf1b5a7 100644
--- a/mysql-test/t/stat_tables.test
+++ b/mysql-test/t/stat_tables.test
@@ -305,4 +305,10 @@ drop database db1;
 drop database db2;
 drop table t1;
 
+--echo #
+--echo # MDEV-16552: [10.0] ASAN global-buffer-overflow in is_stat_table / statistics_for_tables_is_needed
+--echo #
+
+SET use_stat_tables = PREFERABLY;
+SELECT CONVERT_TZ( '1991-09-20 10:11:02', '+00:00', 'GMT' );
 set use_stat_tables=@save_use_stat_tables;
diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc
index 1febc02b903..be4547a69df 100644
--- a/sql/sql_statistics.cc
+++ b/sql/sql_statistics.cc
@@ -3858,11 +3858,11 @@ bool is_stat_table(const char *db, const char *table)
 {
   DBUG_ASSERT(db && table);
 
-  if (!memcmp(db, stat_tables_db_name.str, stat_tables_db_name.length))
+  if (!my_strcasecmp(table_alias_charset, db, stat_tables_db_name.str))
   {
     for (uint i= 0; i < STATISTICS_TABLES; i ++)
     {
-      if (!memcmp(table, stat_table_name[i].str, stat_table_name[i].length))
+      if (!my_strcasecmp(table_alias_charset, table, stat_table_name[i].str))
         return true;
     }
   }

From c89bb15c31f98d2d368414c7366ce61955b70b44 Mon Sep 17 00:00:00 2001
From: Igor Babaev <igor@askmonty.org>
Date: Fri, 13 Jul 2018 17:48:30 -0700
Subject: [PATCH 030/164] MDEV-16757 Memory leak after adding manually min/max
 statistical data            for blob column

ANALYZE TABLE <table> does not collect statistical data on min/max values
for BLOB columns of <table>. However these values can be added into
mysql.column_stats manually by executing proper statements.
Unfortunately this led to a memory leak because the memory allocated
for these values was never freed.
This patch provides the server with a function to free memory allocated
for min/max statistical values of BLOB types.
---
 mysql-test/r/stat_tables.result        | 28 ++++++++++++++++++++++
 mysql-test/r/stat_tables_innodb.result | 28 ++++++++++++++++++++++
 mysql-test/t/stat_tables.test          | 24 +++++++++++++++++++
 sql/sql_statistics.cc                  | 33 ++++++++++++++++++++++++++
 sql/sql_statistics.h                   |  1 +
 sql/table_cache.cc                     |  2 ++
 6 files changed, 116 insertions(+)

diff --git a/mysql-test/r/stat_tables.result b/mysql-test/r/stat_tables.result
index 4a608089d7d..e6b675060a5 100644
--- a/mysql-test/r/stat_tables.result
+++ b/mysql-test/r/stat_tables.result
@@ -524,3 +524,31 @@ SELECT CONVERT_TZ( '1991-09-20 10:11:02', '+00:00', 'GMT' );
 CONVERT_TZ( '1991-09-20 10:11:02', '+00:00', 'GMT' )
 NULL
 set use_stat_tables=@save_use_stat_tables;
+#
+# MDEV-16757: manual addition of min/max statistics for BLOB
+#
+SET use_stat_tables= PREFERABLY;
+CREATE TABLE t1 (pk INT PRIMARY KEY, t TEXT);
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+SELECT * FROM mysql.column_stats;
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+test	t1	pk	1	2	0.0000	4.0000	1.0000	0	NULL	NULL
+test	t1	t	NULL	NULL	0.0000	3.0000	NULL	NULL	NULL	NULL
+DELETE FROM mysql.column_stats
+WHERE db_name='test' AND table_name='t1' AND column_name='t';
+INSERT INTO mysql.column_stats VALUES
+('test','t1','t','bar','foo', 0.0, 3.0, 1.0, 0, NULL, NULL);
+SELECT * FROM mysql.column_stats;
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+test	t1	pk	1	2	0.0000	4.0000	1.0000	0	NULL	NULL
+test	t1	t	bar	foo	0.0000	3.0000	1.0000	0	NULL	NULL
+SELECT pk FROM t1;
+pk
+1
+2
+DROP TABLE t1;
+set use_stat_tables=@save_use_stat_tables;
diff --git a/mysql-test/r/stat_tables_innodb.result b/mysql-test/r/stat_tables_innodb.result
index 87049b228e5..04d73868cfb 100644
--- a/mysql-test/r/stat_tables_innodb.result
+++ b/mysql-test/r/stat_tables_innodb.result
@@ -551,5 +551,33 @@ SELECT CONVERT_TZ( '1991-09-20 10:11:02', '+00:00', 'GMT' );
 CONVERT_TZ( '1991-09-20 10:11:02', '+00:00', 'GMT' )
 NULL
 set use_stat_tables=@save_use_stat_tables;
+#
+# MDEV-16757: manual addition of min/max statistics for BLOB
+#
+SET use_stat_tables= PREFERABLY;
+CREATE TABLE t1 (pk INT PRIMARY KEY, t TEXT);
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+SELECT * FROM mysql.column_stats;
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+test	t1	pk	1	2	0.0000	4.0000	1.0000	0	NULL	NULL
+test	t1	t	NULL	NULL	0.0000	3.0000	NULL	NULL	NULL	NULL
+DELETE FROM mysql.column_stats
+WHERE db_name='test' AND table_name='t1' AND column_name='t';
+INSERT INTO mysql.column_stats VALUES
+('test','t1','t','bar','foo', 0.0, 3.0, 1.0, 0, NULL, NULL);
+SELECT * FROM mysql.column_stats;
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+test	t1	pk	1	2	0.0000	4.0000	1.0000	0	NULL	NULL
+test	t1	t	bar	foo	0.0000	3.0000	1.0000	0	NULL	NULL
+SELECT pk FROM t1;
+pk
+1
+2
+DROP TABLE t1;
+set use_stat_tables=@save_use_stat_tables;
 set optimizer_switch=@save_optimizer_switch_for_stat_tables_test;
 SET SESSION STORAGE_ENGINE=DEFAULT;
diff --git a/mysql-test/t/stat_tables.test b/mysql-test/t/stat_tables.test
index 9f94cf1b5a7..843e6f8aa0b 100644
--- a/mysql-test/t/stat_tables.test
+++ b/mysql-test/t/stat_tables.test
@@ -312,3 +312,27 @@ drop table t1;
 SET use_stat_tables = PREFERABLY;
 SELECT CONVERT_TZ( '1991-09-20 10:11:02', '+00:00', 'GMT' );
 set use_stat_tables=@save_use_stat_tables;
+
+--echo #
+--echo # MDEV-16757: manual addition of min/max statistics for BLOB
+--echo #
+
+SET use_stat_tables= PREFERABLY;
+
+CREATE TABLE t1 (pk INT PRIMARY KEY, t TEXT);
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+ANALYZE TABLE t1;
+--sorted_result
+SELECT * FROM mysql.column_stats;
+DELETE FROM mysql.column_stats
+  WHERE db_name='test' AND table_name='t1' AND column_name='t';
+INSERT INTO mysql.column_stats VALUES
+  ('test','t1','t','bar','foo', 0.0, 3.0, 1.0, 0, NULL, NULL);
+--sorted_result
+SELECT * FROM mysql.column_stats;
+
+SELECT pk FROM t1;
+
+DROP TABLE t1;
+
+set use_stat_tables=@save_use_stat_tables;
diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc
index be4547a69df..537ede91710 100644
--- a/sql/sql_statistics.cc
+++ b/sql/sql_statistics.cc
@@ -2916,6 +2916,39 @@ int read_statistics_for_table(THD *thd, TABLE *table, TABLE_LIST *stat_tables)
 }
 
 
+/**
+  @breif
+  Cleanup of min/max statistical values for table share
+*/
+
+void delete_stat_values_for_table_share(TABLE_SHARE *table_share)
+{
+  TABLE_STATISTICS_CB *stats_cb= &table_share->stats_cb;
+  Table_statistics *table_stats= stats_cb->table_stats;
+  if (!table_stats)
+    return;
+  Column_statistics *column_stats= table_stats->column_stats;
+  if (!column_stats)
+    return;
+
+  for (Field **field_ptr= table_share->field;
+       *field_ptr;
+       field_ptr++, column_stats++)
+  {
+    if (column_stats->min_value)
+    {
+      delete column_stats->min_value;
+      column_stats->min_value= NULL;
+    }
+    if (column_stats->max_value)
+    {
+      delete column_stats->max_value;
+      column_stats->max_value= NULL;
+    }
+  }
+}
+
+
 /**
   @brief
   Check whether any statistics is to be read for tables from a table list
diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h
index 20b2eb66449..6a43e42ab96 100644
--- a/sql/sql_statistics.h
+++ b/sql/sql_statistics.h
@@ -89,6 +89,7 @@ int read_statistics_for_tables_if_needed(THD *thd, TABLE_LIST *tables);
 int collect_statistics_for_table(THD *thd, TABLE *table);
 int alloc_statistics_for_table_share(THD* thd, TABLE_SHARE *share,
                                      bool is_safe);
+void delete_stat_values_for_table_share(TABLE_SHARE *table_share);
 int alloc_statistics_for_table(THD *thd, TABLE *table);
 int update_statistics_for_table(THD *thd, TABLE *table);
 int delete_statistics_for_table(THD *thd, LEX_STRING *db, LEX_STRING *tab);
diff --git a/sql/table_cache.cc b/sql/table_cache.cc
index bdb7914c32b..a31068c9bc3 100644
--- a/sql/table_cache.cc
+++ b/sql/table_cache.cc
@@ -52,6 +52,7 @@
 #include "hash.h"
 #include "table.h"
 #include "sql_base.h"
+#include "sql_statistics.h"
 
 /** Configuration. */
 ulong tdc_size; /**< Table definition cache threshold for LRU eviction. */
@@ -869,6 +870,7 @@ void tdc_release_share(TABLE_SHARE *share)
   mysql_mutex_lock(&share->tdc.LOCK_table_share);
   if (share->tdc.flushed)
   {
+    delete_stat_values_for_table_share(share);
     mysql_mutex_unlock(&share->tdc.LOCK_table_share);
     mysql_mutex_unlock(&LOCK_unused_shares);
     tdc_delete_share_from_hash(share);

From 1fd84f9129f2ed98706f6e225b06b16a13d0ebd0 Mon Sep 17 00:00:00 2001
From: Igor Babaev <igor@askmonty.org>
Date: Fri, 13 Jul 2018 23:03:57 -0700
Subject: [PATCH 031/164] MDEV-16760 CREATE OR REPLACE TABLE never updates
 statistical tables

If the command CREATE OR REPLACE TABLE really replaces a table then
it should remove all data on this table from all statistical tables.
---
 mysql-test/r/stat_tables.result        | 25 +++++++++++++++++++++++++
 mysql-test/r/stat_tables_innodb.result | 25 +++++++++++++++++++++++++
 mysql-test/t/stat_tables.test          | 20 ++++++++++++++++++++
 sql/sql_table.cc                       |  4 ++++
 4 files changed, 74 insertions(+)

diff --git a/mysql-test/r/stat_tables.result b/mysql-test/r/stat_tables.result
index e6b675060a5..c1457d5e91a 100644
--- a/mysql-test/r/stat_tables.result
+++ b/mysql-test/r/stat_tables.result
@@ -552,3 +552,28 @@ pk
 2
 DROP TABLE t1;
 set use_stat_tables=@save_use_stat_tables;
+#
+# MDEV-16760: CREATE OR REPLACE TABLE after ANALYZE TABLE
+#
+SET use_stat_tables= PREFERABLY;
+CREATE TABLE t1 (pk int PRIMARY KEY, c varchar(32));
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+SELECT * FROM t1;
+pk	c
+1	foo
+2	bar
+SELECT * FROM mysql.column_stats;
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+test	t1	pk	1	2	0.0000	4.0000	1.0000	0	NULL	NULL
+test	t1	c	bar	foo	0.0000	3.0000	1.0000	0	NULL	NULL
+CREATE OR REPLACE TABLE t1 (pk int PRIMARY KEY, a char(7));
+SELECT * FROM t1;
+pk	a
+SELECT * FROM mysql.column_stats;
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+DROP TABLE t1;
+set use_stat_tables=@save_use_stat_tables;
diff --git a/mysql-test/r/stat_tables_innodb.result b/mysql-test/r/stat_tables_innodb.result
index 04d73868cfb..2ac868e9341 100644
--- a/mysql-test/r/stat_tables_innodb.result
+++ b/mysql-test/r/stat_tables_innodb.result
@@ -579,5 +579,30 @@ pk
 2
 DROP TABLE t1;
 set use_stat_tables=@save_use_stat_tables;
+#
+# MDEV-16760: CREATE OR REPLACE TABLE after ANALYZE TABLE
+#
+SET use_stat_tables= PREFERABLY;
+CREATE TABLE t1 (pk int PRIMARY KEY, c varchar(32));
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+SELECT * FROM t1;
+pk	c
+1	foo
+2	bar
+SELECT * FROM mysql.column_stats;
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+test	t1	pk	1	2	0.0000	4.0000	1.0000	0	NULL	NULL
+test	t1	c	bar	foo	0.0000	3.0000	1.0000	0	NULL	NULL
+CREATE OR REPLACE TABLE t1 (pk int PRIMARY KEY, a char(7));
+SELECT * FROM t1;
+pk	a
+SELECT * FROM mysql.column_stats;
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+DROP TABLE t1;
+set use_stat_tables=@save_use_stat_tables;
 set optimizer_switch=@save_optimizer_switch_for_stat_tables_test;
 SET SESSION STORAGE_ENGINE=DEFAULT;
diff --git a/mysql-test/t/stat_tables.test b/mysql-test/t/stat_tables.test
index 843e6f8aa0b..d69b00618ea 100644
--- a/mysql-test/t/stat_tables.test
+++ b/mysql-test/t/stat_tables.test
@@ -336,3 +336,23 @@ SELECT pk FROM t1;
 DROP TABLE t1;
 
 set use_stat_tables=@save_use_stat_tables;
+
+--echo #
+--echo # MDEV-16760: CREATE OR REPLACE TABLE after ANALYZE TABLE
+--echo #
+
+SET use_stat_tables= PREFERABLY;
+
+CREATE TABLE t1 (pk int PRIMARY KEY, c varchar(32));
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+ANALYZE TABLE t1;
+SELECT * FROM t1;
+SELECT * FROM mysql.column_stats;
+
+CREATE OR REPLACE TABLE t1 (pk int PRIMARY KEY, a char(7));
+SELECT * FROM t1;
+SELECT * FROM mysql.column_stats;
+
+DROP TABLE t1;
+
+set use_stat_tables=@save_use_stat_tables;
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index 6c71067d51b..ee02b5fc7aa 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -4762,6 +4762,10 @@ int create_table_impl(THD *thd,
     {
       if (create_info->options & HA_LEX_CREATE_REPLACE)
       {
+        LEX_STRING db_name= {(char *) db, strlen(db)};
+        LEX_STRING tab_name= {(char *) table_name, strlen(table_name)};
+        (void) delete_statistics_for_table(thd, &db_name, &tab_name);
+
         TABLE_LIST table_list;
         table_list.init_one_table(db, strlen(db), table_name,
                                   strlen(table_name), table_name,

From 095dc81158902380b8618338efabb5ce480dbd79 Mon Sep 17 00:00:00 2001
From: Igor Babaev <igor@askmonty.org>
Date: Sun, 15 Jul 2018 16:24:24 -0700
Subject: [PATCH 032/164] MDEV-16757 Memory leak after adding manually min/max
 statistical data            for blob column

ANALYZE TABLE <table> does not collect statistical data on min/max values
for BLOB columns of <table>. However these values can be added into
mysql.column_stats manually by executing proper statements.
Unfortunately this led to a memory leak because the memory allocated
for these values was never freed.
This patch provides the server with a function to free memory allocated
for min/max statistical values of BLOB types.

Temporarily changed the test case until MDEV-16711 is fixed as without
this fix the test case for MDEV-16757 did not fail only for 10.0.
---
 mysql-test/r/stat_tables.result        | 24 +++++++++++++++++++
 mysql-test/r/stat_tables_innodb.result | 24 +++++++++++++++++++
 mysql-test/t/stat_tables.test          | 24 +++++++++++++++++++
 sql/sql_statistics.cc                  | 33 ++++++++++++++++++++++++++
 sql/sql_statistics.h                   |  1 +
 sql/table_cache.cc                     |  2 ++
 6 files changed, 108 insertions(+)

diff --git a/mysql-test/r/stat_tables.result b/mysql-test/r/stat_tables.result
index fcced761283..f299603f8f1 100644
--- a/mysql-test/r/stat_tables.result
+++ b/mysql-test/r/stat_tables.result
@@ -517,3 +517,27 @@ drop database db1;
 drop database db2;
 drop table t1;
 set use_stat_tables=@save_use_stat_tables;
+#
+# MDEV-16757: manual addition of min/max statistics for BLOB
+#
+SET use_stat_tables= PREFERABLY;
+CREATE TABLE t1 (pk INT PRIMARY KEY, t TEXT);
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	Warning	Engine-independent statistics are not collected for column 't'
+test.t1	analyze	status	OK
+SELECT * FROM mysql.column_stats;
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+test	t1	pk	1	2	0.0000	4.0000	1.0000	0	NULL	NULL
+DELETE FROM mysql.column_stats
+WHERE db_name='test' AND table_name='t1' AND column_name='t';
+INSERT INTO mysql.column_stats VALUES
+('test','t1','t','bar','foo', 0.0, 3.0, 1.0, 0, NULL, NULL);
+SELECT * FROM mysql.column_stats;
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+test	t1	pk	1	2	0.0000	4.0000	1.0000	0	NULL	NULL
+test	t1	t	bar	foo	0.0000	3.0000	1.0000	0	NULL	NULL
+DROP TABLE t1;
+set use_stat_tables=@save_use_stat_tables;
diff --git a/mysql-test/r/stat_tables_innodb.result b/mysql-test/r/stat_tables_innodb.result
index 0e866755532..d2d92966d2b 100644
--- a/mysql-test/r/stat_tables_innodb.result
+++ b/mysql-test/r/stat_tables_innodb.result
@@ -544,5 +544,29 @@ drop database db1;
 drop database db2;
 drop table t1;
 set use_stat_tables=@save_use_stat_tables;
+#
+# MDEV-16757: manual addition of min/max statistics for BLOB
+#
+SET use_stat_tables= PREFERABLY;
+CREATE TABLE t1 (pk INT PRIMARY KEY, t TEXT);
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	Warning	Engine-independent statistics are not collected for column 't'
+test.t1	analyze	status	OK
+SELECT * FROM mysql.column_stats;
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+test	t1	pk	1	2	0.0000	4.0000	1.0000	0	NULL	NULL
+DELETE FROM mysql.column_stats
+WHERE db_name='test' AND table_name='t1' AND column_name='t';
+INSERT INTO mysql.column_stats VALUES
+('test','t1','t','bar','foo', 0.0, 3.0, 1.0, 0, NULL, NULL);
+SELECT * FROM mysql.column_stats;
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+test	t1	pk	1	2	0.0000	4.0000	1.0000	0	NULL	NULL
+test	t1	t	bar	foo	0.0000	3.0000	1.0000	0	NULL	NULL
+DROP TABLE t1;
+set use_stat_tables=@save_use_stat_tables;
 set optimizer_switch=@save_optimizer_switch_for_stat_tables_test;
 SET SESSION STORAGE_ENGINE=DEFAULT;
diff --git a/mysql-test/t/stat_tables.test b/mysql-test/t/stat_tables.test
index 4cbaa9e27c8..5bbd8cadfbb 100644
--- a/mysql-test/t/stat_tables.test
+++ b/mysql-test/t/stat_tables.test
@@ -306,3 +306,27 @@ drop database db2;
 drop table t1;
 
 set use_stat_tables=@save_use_stat_tables;
+
+--echo #
+--echo # MDEV-16757: manual addition of min/max statistics for BLOB
+--echo #
+
+SET use_stat_tables= PREFERABLY;
+
+CREATE TABLE t1 (pk INT PRIMARY KEY, t TEXT);
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+ANALYZE TABLE t1;
+--sorted_result
+SELECT * FROM mysql.column_stats;
+DELETE FROM mysql.column_stats
+  WHERE db_name='test' AND table_name='t1' AND column_name='t';
+INSERT INTO mysql.column_stats VALUES
+  ('test','t1','t','bar','foo', 0.0, 3.0, 1.0, 0, NULL, NULL);
+--sorted_result
+SELECT * FROM mysql.column_stats;
+
+# SELECT pk FROM t1;
+
+DROP TABLE t1;
+
+set use_stat_tables=@save_use_stat_tables;
diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc
index abf3975dd9c..a1c21421c59 100644
--- a/sql/sql_statistics.cc
+++ b/sql/sql_statistics.cc
@@ -3055,6 +3055,39 @@ int read_statistics_for_table(THD *thd, TABLE *table, TABLE_LIST *stat_tables)
 }
 
 
+/**
+  @breif
+  Cleanup of min/max statistical values for table share
+*/
+
+void delete_stat_values_for_table_share(TABLE_SHARE *table_share)
+{
+  TABLE_STATISTICS_CB *stats_cb= &table_share->stats_cb;
+  Table_statistics *table_stats= stats_cb->table_stats;
+  if (!table_stats)
+    return;
+  Column_statistics *column_stats= table_stats->column_stats;
+  if (!column_stats)
+    return;
+
+  for (Field **field_ptr= table_share->field;
+       *field_ptr;
+       field_ptr++, column_stats++)
+  {
+    if (column_stats->min_value)
+    {
+      delete column_stats->min_value;
+      column_stats->min_value= NULL;
+    }
+    if (column_stats->max_value)
+    {
+      delete column_stats->max_value;
+      column_stats->max_value= NULL;
+    }
+  }
+}
+
+
 /**
   @brief
   Check whether any statistics is to be read for tables from a table list
diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h
index f46583839d1..0611c021e88 100644
--- a/sql/sql_statistics.h
+++ b/sql/sql_statistics.h
@@ -92,6 +92,7 @@ int read_statistics_for_tables_if_needed(THD *thd, TABLE_LIST *tables);
 int collect_statistics_for_table(THD *thd, TABLE *table);
 int alloc_statistics_for_table_share(THD* thd, TABLE_SHARE *share,
                                      bool is_safe);
+void delete_stat_values_for_table_share(TABLE_SHARE *table_share);
 int alloc_statistics_for_table(THD *thd, TABLE *table);
 int update_statistics_for_table(THD *thd, TABLE *table);
 int delete_statistics_for_table(THD *thd, LEX_STRING *db, LEX_STRING *tab);
diff --git a/sql/table_cache.cc b/sql/table_cache.cc
index 16a47b37417..b911252019f 100644
--- a/sql/table_cache.cc
+++ b/sql/table_cache.cc
@@ -52,6 +52,7 @@
 #include "lf.h"
 #include "table.h"
 #include "sql_base.h"
+#include "sql_statistics.h"
 
 
 /** Configuration. */
@@ -786,6 +787,7 @@ void tdc_release_share(TABLE_SHARE *share)
   }
   if (share->tdc->flushed || tdc_records() > tdc_size)
   {
+    delete_stat_values_for_table_share(share);
     mysql_mutex_unlock(&LOCK_unused_shares);
     tdc_delete_share_from_hash(share->tdc);
     DBUG_VOID_RETURN;

From ae0eb507bda858a13475faef4476dd496c4dbc01 Mon Sep 17 00:00:00 2001
From: Igor Babaev <igor@askmonty.org>
Date: Fri, 13 Jul 2018 23:03:57 -0700
Subject: [PATCH 033/164] MDEV-16760 CREATE OR REPLACE TABLE never updates
 statistical tables

If the command CREATE OR REPLACE TABLE really replaces a table then
it should remove all data on this table from all statistical tables.
---
 mysql-test/r/stat_tables.result        | 25 +++++++++++++++++++++++++
 mysql-test/r/stat_tables_innodb.result | 25 +++++++++++++++++++++++++
 mysql-test/t/stat_tables.test          | 20 ++++++++++++++++++++
 sql/sql_table.cc                       |  4 ++++
 4 files changed, 74 insertions(+)

diff --git a/mysql-test/r/stat_tables.result b/mysql-test/r/stat_tables.result
index f299603f8f1..1f1cad10c23 100644
--- a/mysql-test/r/stat_tables.result
+++ b/mysql-test/r/stat_tables.result
@@ -541,3 +541,28 @@ test	t1	pk	1	2	0.0000	4.0000	1.0000	0	NULL	NULL
 test	t1	t	bar	foo	0.0000	3.0000	1.0000	0	NULL	NULL
 DROP TABLE t1;
 set use_stat_tables=@save_use_stat_tables;
+#
+# MDEV-16760: CREATE OR REPLACE TABLE after ANALYZE TABLE
+#
+SET use_stat_tables= PREFERABLY;
+CREATE TABLE t1 (pk int PRIMARY KEY, c varchar(32));
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+SELECT * FROM t1;
+pk	c
+1	foo
+2	bar
+SELECT * FROM mysql.column_stats;
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+test	t1	pk	1	2	0.0000	4.0000	1.0000	0	NULL	NULL
+test	t1	c	bar	foo	0.0000	3.0000	1.0000	0	NULL	NULL
+CREATE OR REPLACE TABLE t1 (pk int PRIMARY KEY, a char(7));
+SELECT * FROM t1;
+pk	a
+SELECT * FROM mysql.column_stats;
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+DROP TABLE t1;
+set use_stat_tables=@save_use_stat_tables;
diff --git a/mysql-test/r/stat_tables_innodb.result b/mysql-test/r/stat_tables_innodb.result
index d2d92966d2b..fc1ecdf66c4 100644
--- a/mysql-test/r/stat_tables_innodb.result
+++ b/mysql-test/r/stat_tables_innodb.result
@@ -568,5 +568,30 @@ test	t1	pk	1	2	0.0000	4.0000	1.0000	0	NULL	NULL
 test	t1	t	bar	foo	0.0000	3.0000	1.0000	0	NULL	NULL
 DROP TABLE t1;
 set use_stat_tables=@save_use_stat_tables;
+#
+# MDEV-16760: CREATE OR REPLACE TABLE after ANALYZE TABLE
+#
+SET use_stat_tables= PREFERABLY;
+CREATE TABLE t1 (pk int PRIMARY KEY, c varchar(32));
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+SELECT * FROM t1;
+pk	c
+1	foo
+2	bar
+SELECT * FROM mysql.column_stats;
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+test	t1	pk	1	2	0.0000	4.0000	1.0000	0	NULL	NULL
+test	t1	c	bar	foo	0.0000	3.0000	1.0000	0	NULL	NULL
+CREATE OR REPLACE TABLE t1 (pk int PRIMARY KEY, a char(7));
+SELECT * FROM t1;
+pk	a
+SELECT * FROM mysql.column_stats;
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+DROP TABLE t1;
+set use_stat_tables=@save_use_stat_tables;
 set optimizer_switch=@save_optimizer_switch_for_stat_tables_test;
 SET SESSION STORAGE_ENGINE=DEFAULT;
diff --git a/mysql-test/t/stat_tables.test b/mysql-test/t/stat_tables.test
index 5bbd8cadfbb..b98ca641dc0 100644
--- a/mysql-test/t/stat_tables.test
+++ b/mysql-test/t/stat_tables.test
@@ -330,3 +330,23 @@ SELECT * FROM mysql.column_stats;
 DROP TABLE t1;
 
 set use_stat_tables=@save_use_stat_tables;
+
+--echo #
+--echo # MDEV-16760: CREATE OR REPLACE TABLE after ANALYZE TABLE
+--echo #
+
+SET use_stat_tables= PREFERABLY;
+
+CREATE TABLE t1 (pk int PRIMARY KEY, c varchar(32));
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+ANALYZE TABLE t1;
+SELECT * FROM t1;
+SELECT * FROM mysql.column_stats;
+
+CREATE OR REPLACE TABLE t1 (pk int PRIMARY KEY, a char(7));
+SELECT * FROM t1;
+SELECT * FROM mysql.column_stats;
+
+DROP TABLE t1;
+
+set use_stat_tables=@save_use_stat_tables;
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index 3c9146354d6..9a8c737cd82 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -4770,6 +4770,10 @@ int create_table_impl(THD *thd,
     {
       if (options.or_replace())
       {
+        LEX_STRING db_name= {(char *) db, strlen(db)};
+        LEX_STRING tab_name= {(char *) table_name, strlen(table_name)};
+        (void) delete_statistics_for_table(thd, &db_name, &tab_name);
+
         TABLE_LIST table_list;
         table_list.init_one_table(db, strlen(db), table_name,
                                   strlen(table_name), table_name,

From b75d819604bbb6c461e6363e331bd6bd1ba7a787 Mon Sep 17 00:00:00 2001
From: Igor Babaev <igor@askmonty.org>
Date: Sun, 15 Jul 2018 18:40:25 -0700
Subject: [PATCH 034/164] MDEV-16711 Crash in Field_blob::store() while reading
 statistics           for the small InnoDB table

This bug was introduced by the patch 6c414fcf89510215d6d3466eb9992d444eadae89.
The patch has not taken into account that some objects of the Field_* types
are created only for TABLE_SHARE and the field 'table' is set to NULL
for them. In particular such are objects created to store statistical
min/max values for columns.
---
 mysql-test/r/stat_tables.result        | 20 ++++++++++++++++++++
 mysql-test/r/stat_tables_innodb.result | 20 ++++++++++++++++++++
 mysql-test/t/stat_tables.test          | 20 +++++++++++++++++++-
 sql/field.cc                           |  8 +++++++-
 4 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/mysql-test/r/stat_tables.result b/mysql-test/r/stat_tables.result
index 1f1cad10c23..279c09ff44f 100644
--- a/mysql-test/r/stat_tables.result
+++ b/mysql-test/r/stat_tables.result
@@ -539,6 +539,10 @@ SELECT * FROM mysql.column_stats;
 db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
 test	t1	pk	1	2	0.0000	4.0000	1.0000	0	NULL	NULL
 test	t1	t	bar	foo	0.0000	3.0000	1.0000	0	NULL	NULL
+SELECT pk FROM t1;
+pk
+1
+2
 DROP TABLE t1;
 set use_stat_tables=@save_use_stat_tables;
 #
@@ -566,3 +570,19 @@ SELECT * FROM mysql.column_stats;
 db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
 DROP TABLE t1;
 set use_stat_tables=@save_use_stat_tables;
+#
+# MDEV-16711:CREATE OR REPLACE TABLE introducing BLOB column
+#
+SET use_stat_tables= PREFERABLY;
+CREATE TABLE t1 (pk INT PRIMARY KEY, t CHAR(60));
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+CREATE OR REPLACE TABLE t1 (pk INT PRIMARY KEY, t TEXT);
+SELECT MAX(pk) FROM t1;
+MAX(pk)
+NULL
+DROP TABLE t1;
+set use_stat_tables=@save_use_stat_tables;
diff --git a/mysql-test/r/stat_tables_innodb.result b/mysql-test/r/stat_tables_innodb.result
index fc1ecdf66c4..1604c413743 100644
--- a/mysql-test/r/stat_tables_innodb.result
+++ b/mysql-test/r/stat_tables_innodb.result
@@ -566,6 +566,10 @@ SELECT * FROM mysql.column_stats;
 db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
 test	t1	pk	1	2	0.0000	4.0000	1.0000	0	NULL	NULL
 test	t1	t	bar	foo	0.0000	3.0000	1.0000	0	NULL	NULL
+SELECT pk FROM t1;
+pk
+1
+2
 DROP TABLE t1;
 set use_stat_tables=@save_use_stat_tables;
 #
@@ -593,5 +597,21 @@ SELECT * FROM mysql.column_stats;
 db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
 DROP TABLE t1;
 set use_stat_tables=@save_use_stat_tables;
+#
+# MDEV-16711:CREATE OR REPLACE TABLE introducing BLOB column
+#
+SET use_stat_tables= PREFERABLY;
+CREATE TABLE t1 (pk INT PRIMARY KEY, t CHAR(60));
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+CREATE OR REPLACE TABLE t1 (pk INT PRIMARY KEY, t TEXT);
+SELECT MAX(pk) FROM t1;
+MAX(pk)
+NULL
+DROP TABLE t1;
+set use_stat_tables=@save_use_stat_tables;
 set optimizer_switch=@save_optimizer_switch_for_stat_tables_test;
 SET SESSION STORAGE_ENGINE=DEFAULT;
diff --git a/mysql-test/t/stat_tables.test b/mysql-test/t/stat_tables.test
index b98ca641dc0..51c9995b6fa 100644
--- a/mysql-test/t/stat_tables.test
+++ b/mysql-test/t/stat_tables.test
@@ -325,7 +325,7 @@ INSERT INTO mysql.column_stats VALUES
 --sorted_result
 SELECT * FROM mysql.column_stats;
 
-# SELECT pk FROM t1;
+SELECT pk FROM t1;
 
 DROP TABLE t1;
 
@@ -350,3 +350,21 @@ SELECT * FROM mysql.column_stats;
 DROP TABLE t1;
 
 set use_stat_tables=@save_use_stat_tables;
+
+
+--echo #
+--echo # MDEV-16711:CREATE OR REPLACE TABLE introducing BLOB column
+--echo #
+
+SET use_stat_tables= PREFERABLY;
+
+CREATE TABLE t1 (pk INT PRIMARY KEY, t CHAR(60));
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+ANALYZE TABLE t1;
+CREATE OR REPLACE TABLE t1 (pk INT PRIMARY KEY, t TEXT);
+
+SELECT MAX(pk) FROM t1;
+
+DROP TABLE t1;
+
+set use_stat_tables=@save_use_stat_tables;
diff --git a/sql/field.cc b/sql/field.cc
index 64c51677c0f..71fc5684d92 100644
--- a/sql/field.cc
+++ b/sql/field.cc
@@ -7942,7 +7942,13 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs)
     return 0;
   }
 
-  if (table->blob_storage)    // GROUP_CONCAT with ORDER BY | DISTINCT
+  /*
+    For min/max fields of statistical data 'table' is set to NULL.
+    It could not be otherwise as this data is shared by many instances
+    of the same base table.
+  */
+
+  if (table && table->blob_storage)    // GROUP_CONCAT with ORDER BY | DISTINCT
   {
     DBUG_ASSERT(!f_is_hex_escape(flags));
     DBUG_ASSERT(field_charset == cs);

From fe9f2f4bb63b379a532016c28c3771510061a47d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Wed, 6 Jun 2018 14:19:14 +0300
Subject: [PATCH 035/164] MDEV-16401: Apply review comments to MDEV-16005

Do not hold LOCK_thd_data during my_error or WSREP_DEBUG. Similarly,
release LOCK_thd_data before close_thread_tables() call.
---
 sql/sql_parse.cc | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc
index 89b5adff60c..7e72c1a9397 100644
--- a/sql/sql_parse.cc
+++ b/sql/sql_parse.cc
@@ -1271,9 +1271,9 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
     if (thd->wsrep_conflict_state == ABORTED &&
         command != COM_STMT_CLOSE && command != COM_QUIT)
     {
+      mysql_mutex_unlock(&thd->LOCK_thd_data);
       my_error(ER_LOCK_DEADLOCK, MYF(0), "wsrep aborted transaction");
       WSREP_DEBUG("Deadlock error for: %s", thd->query());
-      mysql_mutex_unlock(&thd->LOCK_thd_data);
       thd->reset_killed();
       thd->mysys_var->abort     = 0;
       thd->wsrep_conflict_state = NO_CONFLICT;
@@ -7290,34 +7290,44 @@ static void wsrep_mysql_parse(THD *thd, char *rawbuf, uint length,
             thd->lex->sql_command != SQLCOM_SELECT  &&
             (thd->wsrep_retry_counter < thd->variables.wsrep_retry_autocommit))
         {
-          WSREP_DEBUG("wsrep retrying AC query: %s", 
+          mysql_mutex_unlock(&thd->LOCK_thd_data);
+          WSREP_DEBUG("wsrep retrying AC query: %s",
                       (thd->query()) ? thd->query() : "void");
 
 	  /* Performance Schema Interface instrumentation, end */
 	  MYSQL_END_STATEMENT(thd->m_statement_psi, thd->get_stmt_da());
 	  thd->m_statement_psi= NULL;
           thd->m_digest= NULL;
+	  // Released thd->LOCK_thd_data above as below could end up
+	  // close_thread_tables()/close_open_tables()/close_thread_table()/mysql_mutex_lock(&thd->LOCK_thd_data)
           close_thread_tables(thd);
 
+	  mysql_mutex_lock(&thd->LOCK_thd_data);
           thd->wsrep_conflict_state= RETRY_AUTOCOMMIT;
           thd->wsrep_retry_counter++;            // grow
           wsrep_copy_query(thd);
           thd->set_time();
           parser_state->reset(rawbuf, length);
+          mysql_mutex_unlock(&thd->LOCK_thd_data);
         }
         else
         {
-          WSREP_DEBUG("%s, thd: %lu is_AC: %d, retry: %lu - %lu SQL: %s", 
-                      (thd->wsrep_conflict_state == ABORTED) ? 
+          mysql_mutex_unlock(&thd->LOCK_thd_data);
+	  // This does dirty read to wsrep variables but it is only a debug code
+          WSREP_DEBUG("%s, thd: %lu is_AC: %d, retry: %lu - %lu SQL: %s",
+                      (thd->wsrep_conflict_state == ABORTED) ?
                       "BF Aborted" : "cert failure",
-                      thd->thread_id, is_autocommit, thd->wsrep_retry_counter, 
+                      thd->thread_id, is_autocommit, thd->wsrep_retry_counter,
                       thd->variables.wsrep_retry_autocommit, thd->query());
           my_error(ER_LOCK_DEADLOCK, MYF(0), "wsrep aborted transaction");
+
+	  mysql_mutex_lock(&thd->LOCK_thd_data);
           thd->wsrep_conflict_state= NO_CONFLICT;
           if (thd->wsrep_conflict_state != REPLAYING)
             thd->wsrep_retry_counter= 0;             //  reset
+	  mysql_mutex_unlock(&thd->LOCK_thd_data);
         }
-        mysql_mutex_unlock(&thd->LOCK_thd_data);
+
         thd->reset_killed();
       }
       else
@@ -7353,6 +7363,7 @@ static void wsrep_mysql_parse(THD *thd, char *rawbuf, uint length,
 #endif /* WITH_WSREP */
 }
 
+
 /*
   When you modify mysql_parse(), you may need to modify
   mysql_test_parse_for_slave() in this same file.

From 8c45eb3ea51e80b429c789713fa85ef380cecdd3 Mon Sep 17 00:00:00 2001
From: Daniel Black <grooverdan@users.sourceforge.net>
Date: Sat, 13 Jan 2018 12:27:28 +1100
Subject: [PATCH 036/164] MDEV-15050 scripts: mysql_install_db.{sh|pl},
 mysqld_multi - mysqld is in @sbindir@

Closes #551
---
 scripts/mysql_install_db.pl.in | 2 +-
 scripts/mysql_install_db.sh    | 2 +-
 scripts/mysqld_multi.sh        | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/mysql_install_db.pl.in b/scripts/mysql_install_db.pl.in
index c6714f66bff..2a65dc94ba8 100644
--- a/scripts/mysql_install_db.pl.in
+++ b/scripts/mysql_install_db.pl.in
@@ -326,7 +326,7 @@ else
   $opt->{basedir} = '@prefix@';
   $bindir         = '@bindir@';
   $extra_bindir   = $bindir;
-  $mysqld         = '@libexecdir@/mysqld';
+  $mysqld         = '@sbindir@/mysqld';
   $pkgdatadir     = '@pkgdatadir@';
   $scriptdir      = '@scriptdir@';
 }
diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh
index a8f32526a7e..0272a19931f 100644
--- a/scripts/mysql_install_db.sh
+++ b/scripts/mysql_install_db.sh
@@ -311,7 +311,7 @@ else
   basedir="@prefix@"
   bindir="@bindir@"
   resolveip="$bindir/resolveip"
-  mysqld="@libexecdir@/mysqld"
+  mysqld="@sbindir@/mysqld"
   pkgdatadir="@pkgdatadir@"
 fi
 
diff --git a/scripts/mysqld_multi.sh b/scripts/mysqld_multi.sh
index e96d12e3f89..8f5adfe4842 100644
--- a/scripts/mysqld_multi.sh
+++ b/scripts/mysqld_multi.sh
@@ -30,7 +30,7 @@ $opt_example       = 0;
 $opt_help          = 0;
 $opt_log           = undef();
 $opt_mysqladmin    = "@bindir@/mysqladmin";
-$opt_mysqld        = "@libexecdir@/mysqld";
+$opt_mysqld        = "@sbindir@/mysqld";
 $opt_no_log        = 0;
 $opt_password      = undef();
 $opt_tcp_ip        = 0;

From ada54101a7185782657813c553907f61f2a35faf Mon Sep 17 00:00:00 2001
From: sachin <sachin.setiya@mariadb.com>
Date: Wed, 30 May 2018 16:25:44 +0530
Subject: [PATCH 037/164] MDEV-9266 Creating index on temporaray table breaks
 replication

Problem:- Create/drop index was logged into binlog.

Goal:- Operation on temporary table should not be binlog when binlog format
is row.

Solution:-
We should add CF_FORCE_ORIGINAL_BINLOG_FORMAT when there is ddl on temp
table.
For optimize, analyze, repair we wont change anything ,Then will
be logged in binlog , But they also dont throw any error if operation fails
Since slave wont be having any temp table , but these operation on tmp
table will be processed without breaking replication.

For rename we need a different logic MDEV-16728 will solve it.
---
 .../binlog/include/check_binlog_size.inc      | 31 +++++++++++++++++++
 .../binlog/r/binlog_tmp_table_row.result      |  7 +++++
 .../suite/binlog/t/binlog_tmp_table_row.test  | 30 ++++++++++++++++++
 sql/sql_parse.cc                              |  3 ++
 4 files changed, 71 insertions(+)
 create mode 100644 mysql-test/suite/binlog/include/check_binlog_size.inc
 create mode 100644 mysql-test/suite/binlog/r/binlog_tmp_table_row.result
 create mode 100644 mysql-test/suite/binlog/t/binlog_tmp_table_row.test

diff --git a/mysql-test/suite/binlog/include/check_binlog_size.inc b/mysql-test/suite/binlog/include/check_binlog_size.inc
new file mode 100644
index 00000000000..9df161ec843
--- /dev/null
+++ b/mysql-test/suite/binlog/include/check_binlog_size.inc
@@ -0,0 +1,31 @@
+# This file runs the query and checks
+# whether the size of binlog is increased or not
+# If size is changed it issue die command
+# Parameters
+# $sql_query = query to run
+
+#Only last row of show binlog events matter
+--let $tmp= 0
+--let $counter= 1
+while ($tmp != "No such row")
+{
+  --let $initial_binlog_size= $tmp
+  --let $tmp= query_get_value(show binary logs, File_size, $counter)
+  --inc $counter
+}
+
+--eval  $sql_query
+
+--let $tmp= 0
+--let $counter= 1
+while ($tmp != "No such row")
+{
+  --let $current_binlog_size= $tmp
+  --let $tmp= query_get_value(show binary logs, File_size, $counter)
+  --inc $counter
+}
+
+if ($initial_binlog_size != $current_binlog_size)
+{
+  die "Binlog size changed";
+}
diff --git a/mysql-test/suite/binlog/r/binlog_tmp_table_row.result b/mysql-test/suite/binlog/r/binlog_tmp_table_row.result
new file mode 100644
index 00000000000..71bd75d89cb
--- /dev/null
+++ b/mysql-test/suite/binlog/r/binlog_tmp_table_row.result
@@ -0,0 +1,7 @@
+RESET MASTER;
+#Create table test
+create temporary table t1(a int, b int);
+#Add index test
+create index index_a on t1(a);
+#drop index test
+drop index index_a on t1;
diff --git a/mysql-test/suite/binlog/t/binlog_tmp_table_row.test b/mysql-test/suite/binlog/t/binlog_tmp_table_row.test
new file mode 100644
index 00000000000..ce11c880679
--- /dev/null
+++ b/mysql-test/suite/binlog/t/binlog_tmp_table_row.test
@@ -0,0 +1,30 @@
+# ==== Purpose ====
+#
+# Test if statements used temporary tables are not binlogged in the case of
+# binlog_format=row
+#
+# ==== Method ====
+#
+# We will see if binlog file size is increased or not, It should be constant for the
+# entire period of test.
+#
+# ==== Related bugs ====
+#
+# Mdev-9266
+#
+source include/have_log_bin.inc;
+source include/have_binlog_format_row.inc;
+
+RESET MASTER;
+
+--echo #Create table test
+--let $sql_query= create temporary table t1(a int, b int)
+--source suite/binlog/include/check_binlog_size.inc
+
+--echo #Add index test
+--let $sql_query= create index index_a on t1(a)
+--source suite/binlog/include/check_binlog_size.inc
+
+--echo #drop index test
+--let $sql_query= drop index index_a on t1
+--source suite/binlog/include/check_binlog_size.inc
diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc
index 28d11ef2e5c..a9849c7248d 100644
--- a/sql/sql_parse.cc
+++ b/sql/sql_parse.cc
@@ -457,6 +457,9 @@ void init_update_queries(void)
   sql_command_flags[SQLCOM_TRUNCATE]|= CF_FORCE_ORIGINAL_BINLOG_FORMAT;
   /* We don't want to replicate DROP for temp tables in row format */
   sql_command_flags[SQLCOM_DROP_TABLE]|= CF_FORCE_ORIGINAL_BINLOG_FORMAT;
+  /* We don't want to replicate CREATE/DROP INDEX for temp tables in row format */
+  sql_command_flags[SQLCOM_CREATE_INDEX]|= CF_FORCE_ORIGINAL_BINLOG_FORMAT;
+  sql_command_flags[SQLCOM_DROP_INDEX]|= CF_FORCE_ORIGINAL_BINLOG_FORMAT;
   /* One can change replication mode with SET */
   sql_command_flags[SQLCOM_SET_OPTION]|= CF_FORCE_ORIGINAL_BINLOG_FORMAT;
 

From 4d06b7e1bd3b825da32c9200e6f5ca609add0e13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Wed, 18 Jul 2018 11:22:17 +0300
Subject: [PATCH 038/164] MDEV-16769: Notes "WSREP: Waiting for SST to
 complete" flood the error log

Used wrong initialization for condition timeout, should have
used set_timespec.
---
 sql/wsrep_sst.cc | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc
index 60683bf740c..155051bfe84 100644
--- a/sql/wsrep_sst.cc
+++ b/sql/wsrep_sst.cc
@@ -204,8 +204,7 @@ void wsrep_sst_grab ()
 // Wait for end of SST
 bool wsrep_sst_wait ()
 {
-  struct timespec wtime = {WSREP_TIMEDWAIT_SECONDS, 0};
-  uint32 total_wtime = 0;
+  double total_wtime = 0;
 
   if (mysql_mutex_lock (&LOCK_wsrep_sst))
     abort();
@@ -214,14 +213,18 @@ bool wsrep_sst_wait ()
 
   while (!sst_complete)
   {
+    struct timespec wtime;
+    set_timespec(wtime, WSREP_TIMEDWAIT_SECONDS);
+    time_t start_time = time(NULL);
     mysql_cond_timedwait (&COND_wsrep_sst, &LOCK_wsrep_sst, &wtime);
+    time_t end_time = time(NULL);
 
     if (!sst_complete)
     {
-      total_wtime += wtime.tv_sec;
-      WSREP_DEBUG("Waiting for SST to complete. waited %u secs.", total_wtime);
+      total_wtime += difftime(end_time, start_time);
+      WSREP_DEBUG("Waiting for SST to complete. current seqno: %ld waited %f secs.", local_seqno, total_wtime);
       service_manager_extend_timeout(WSREP_EXTEND_TIMEOUT_INTERVAL,
-        "WSREP state transfer ongoing, current seqno: %ld", local_seqno);
+        "WSREP state transfer ongoing, current seqno: %ld waited %f secs", local_seqno, total_wtime);
     }
   }
 
@@ -1319,19 +1322,22 @@ void wsrep_SE_init_grab()
 
 void wsrep_SE_init_wait()
 {
-  struct timespec wtime = {WSREP_TIMEDWAIT_SECONDS, 0};
-  uint32 total_wtime=0;
+  double total_wtime=0;
 
   while (SE_initialized == false)
   {
+    struct timespec wtime;
+    set_timespec(wtime, WSREP_TIMEDWAIT_SECONDS);
+    time_t start_time = time(NULL);
     mysql_cond_timedwait (&COND_wsrep_sst_init, &LOCK_wsrep_sst_init, &wtime);
+    time_t end_time = time(NULL);
 
     if (!SE_initialized)
     {
-      total_wtime += wtime.tv_sec;
-      WSREP_DEBUG("Waiting for SST to complete. waited %u secs.", total_wtime);
+      total_wtime += difftime(end_time, start_time);
+      WSREP_DEBUG("Waiting for SST to complete. current seqno: %ld waited %f secs.", local_seqno, total_wtime);
       service_manager_extend_timeout(WSREP_EXTEND_TIMEOUT_INTERVAL,
-        "WSREP SE initialization ongoing.");
+        "WSREP state transfer ongoing, current seqno: %ld waited %f secs", local_seqno, total_wtime);
     }
   }
 

From ab58493db22d870cb6a470d6b0551cfc70dd09f3 Mon Sep 17 00:00:00 2001
From: Alexander Barkov <bar@mariadb.com>
Date: Thu, 19 Jul 2018 09:55:19 +0400
Subject: [PATCH 039/164] MDEV-13118 Wrong results with LOWER and UPPER and
 subquery

This problem is similar to MDEV-10306.

1. Fixing Item_str_conv::val_str(String *str) to return the result in "str",
   and to use tmp_value only as a temporary buffer for args[0]->val_str().
   The new code version now guarantees that the result is always returned in
   "str". The trick with copy_if_not_alloced() is not used any more.

2. The change #1 revealed the same problem in SUBSTRING_INDEX(),
   so some tests with combinations of UPPER()/LOWER() and SUBSTRING_INDEX()
   started to fail. Fixing Item_func_substr_index::val_str() the same way,
   to return the result in "str" and use tmp_value as a temporary buffer
   for args[0]->val_str().
---
 mysql-test/include/ctype_mdev13118.inc |  15 ++++
 mysql-test/r/ctype_binary.result       |  23 ++++++
 mysql-test/r/ctype_eucjpms.result      |  23 ++++++
 mysql-test/r/ctype_euckr.result        |  29 +++++++
 mysql-test/r/ctype_gbk.result          |  23 ++++++
 mysql-test/r/ctype_latin1.result       |  23 ++++++
 mysql-test/r/ctype_ucs.result          |  23 ++++++
 mysql-test/r/ctype_ujis.result         |  23 ++++++
 mysql-test/r/ctype_utf16.result        |  23 ++++++
 mysql-test/r/ctype_utf16le.result      |  29 +++++++
 mysql-test/r/ctype_utf32.result        |  23 ++++++
 mysql-test/r/ctype_utf8.result         |  23 ++++++
 mysql-test/r/ctype_utf8mb4.result      |  23 ++++++
 mysql-test/t/ctype_binary.test         |   3 +
 mysql-test/t/ctype_eucjpms.test        |   2 +
 mysql-test/t/ctype_euckr.test          |  11 +++
 mysql-test/t/ctype_gbk.test            |   3 +
 mysql-test/t/ctype_latin1.test         |   3 +
 mysql-test/t/ctype_ucs.test            |   4 +
 mysql-test/t/ctype_ujis.test           |   4 +
 mysql-test/t/ctype_utf16.test          |   5 ++
 mysql-test/t/ctype_utf16le.test        |  13 +++
 mysql-test/t/ctype_utf32.test          |   8 ++
 mysql-test/t/ctype_utf8.test           |   7 ++
 mysql-test/t/ctype_utf8mb4.test        |   8 ++
 sql/item_strfunc.cc                    | 107 ++++++++++++++-----------
 26 files changed, 433 insertions(+), 48 deletions(-)
 create mode 100644 mysql-test/include/ctype_mdev13118.inc

diff --git a/mysql-test/include/ctype_mdev13118.inc b/mysql-test/include/ctype_mdev13118.inc
new file mode 100644
index 00000000000..efcb57299df
--- /dev/null
+++ b/mysql-test/include/ctype_mdev13118.inc
@@ -0,0 +1,15 @@
+--echo #
+--echo # MDEV-13118 Wrong results with LOWER and UPPER and subquery
+--echo #
+
+SET @save_optimizer_switch=@@optimizer_switch;
+SET optimizer_switch=_latin1'derived_merge=on';
+CREATE TABLE t1 AS SELECT REPEAT('a', 10) AS t LIMIT 0;
+SHOW CREATE TABLE t1;
+INSERT INTO t1 VALUES ('abcdefghi'),('ABCDEFGHI');
+--sorted_result
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT LOWER(t) t2 FROM t1) sub;
+--sorted_result
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT UPPER(t) t2 FROM t1) sub;
+DROP TABLE t1;
+SET optimizer_switch=@save_optimizer_switch;
diff --git a/mysql-test/r/ctype_binary.result b/mysql-test/r/ctype_binary.result
index 48b481883ca..b28d9040547 100644
--- a/mysql-test/r/ctype_binary.result
+++ b/mysql-test/r/ctype_binary.result
@@ -3022,6 +3022,29 @@ DROP TABLE t1;
 SELECT _binary 0x7E, _binary X'7E', _binary B'01111110';
 _binary 0x7E	_binary X'7E'	_binary B'01111110'
 ~	~	~
+SET NAMES utf8, character_set_connection=binary;
+#
+# MDEV-13118 Wrong results with LOWER and UPPER and subquery
+#
+SET @save_optimizer_switch=@@optimizer_switch;
+SET optimizer_switch=_latin1'derived_merge=on';
+CREATE TABLE t1 AS SELECT REPEAT('a', 10) AS t LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `t` varbinary(10) NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('abcdefghi'),('ABCDEFGHI');
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT LOWER(t) t2 FROM t1) sub;
+c2
+ABCDEFGHI-ABCDEFGHI
+abcdefghi-abcdefghi
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT UPPER(t) t2 FROM t1) sub;
+c2
+ABCDEFGHI-ABCDEFGHI
+abcdefghi-abcdefghi
+DROP TABLE t1;
+SET optimizer_switch=@save_optimizer_switch;
 #
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/ctype_eucjpms.result b/mysql-test/r/ctype_eucjpms.result
index a1232c115e9..1e2312638c9 100644
--- a/mysql-test/r/ctype_eucjpms.result
+++ b/mysql-test/r/ctype_eucjpms.result
@@ -33636,6 +33636,29 @@ HEX(a)	CHAR_LENGTH(a)
 DROP TABLE t1;
 SELECT _eucjpms 0x8EA0;
 ERROR HY000: Invalid eucjpms character string: '8EA0'
+SET NAMES eucjpms;
+#
+# MDEV-13118 Wrong results with LOWER and UPPER and subquery
+#
+SET @save_optimizer_switch=@@optimizer_switch;
+SET optimizer_switch=_latin1'derived_merge=on';
+CREATE TABLE t1 AS SELECT REPEAT('a', 10) AS t LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `t` varchar(10) CHARACTER SET eucjpms NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('abcdefghi'),('ABCDEFGHI');
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT LOWER(t) t2 FROM t1) sub;
+c2
+abcdefghi-abcdefghi
+abcdefghi-abcdefghi
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT UPPER(t) t2 FROM t1) sub;
+c2
+ABCDEFGHI-ABCDEFGHI
+ABCDEFGHI-ABCDEFGHI
+DROP TABLE t1;
+SET optimizer_switch=@save_optimizer_switch;
 #
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/ctype_euckr.result b/mysql-test/r/ctype_euckr.result
index dcb68cfe60b..cea93b0c808 100644
--- a/mysql-test/r/ctype_euckr.result
+++ b/mysql-test/r/ctype_euckr.result
@@ -25274,3 +25274,32 @@ A1A1A1A1A1A120202020202020202020202020202020202020
 #
 # End of 5.6 tests
 #
+#
+# Start of 10.0 tests
+#
+SET NAMES utf8, character_set_connection=euckr;
+#
+# MDEV-13118 Wrong results with LOWER and UPPER and subquery
+#
+SET @save_optimizer_switch=@@optimizer_switch;
+SET optimizer_switch=_latin1'derived_merge=on';
+CREATE TABLE t1 AS SELECT REPEAT('a', 10) AS t LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `t` varchar(10) CHARACTER SET euckr NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('abcdefghi'),('ABCDEFGHI');
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT LOWER(t) t2 FROM t1) sub;
+c2
+abcdefghi-abcdefghi
+abcdefghi-abcdefghi
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT UPPER(t) t2 FROM t1) sub;
+c2
+ABCDEFGHI-ABCDEFGHI
+ABCDEFGHI-ABCDEFGHI
+DROP TABLE t1;
+SET optimizer_switch=@save_optimizer_switch;
+#
+# End of 10.0 tests
+#
diff --git a/mysql-test/r/ctype_gbk.result b/mysql-test/r/ctype_gbk.result
index 2d31a15bb62..15f9bc6bfea 100644
--- a/mysql-test/r/ctype_gbk.result
+++ b/mysql-test/r/ctype_gbk.result
@@ -4943,6 +4943,29 @@ E05C5B
 E05B
 DROP TABLE t1;
 # Start of ctype_E05C.inc
+SET NAMES utf8, character_set_connection=gbk;
+#
+# MDEV-13118 Wrong results with LOWER and UPPER and subquery
+#
+SET @save_optimizer_switch=@@optimizer_switch;
+SET optimizer_switch=_latin1'derived_merge=on';
+CREATE TABLE t1 AS SELECT REPEAT('a', 10) AS t LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `t` varchar(10) CHARACTER SET gbk NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('abcdefghi'),('ABCDEFGHI');
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT LOWER(t) t2 FROM t1) sub;
+c2
+abcdefghi-abcdefghi
+abcdefghi-abcdefghi
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT UPPER(t) t2 FROM t1) sub;
+c2
+ABCDEFGHI-ABCDEFGHI
+ABCDEFGHI-ABCDEFGHI
+DROP TABLE t1;
+SET optimizer_switch=@save_optimizer_switch;
 #
 # MDEV-9886 Illegal mix of collations with a view comparing a field to a binary constant
 #
diff --git a/mysql-test/r/ctype_latin1.result b/mysql-test/r/ctype_latin1.result
index cf48aaab09b..8913c8082c8 100644
--- a/mysql-test/r/ctype_latin1.result
+++ b/mysql-test/r/ctype_latin1.result
@@ -7939,6 +7939,29 @@ a
 0
 DROP VIEW v1;
 DROP TABLE t1;
+SET NAMES latin1;
+#
+# MDEV-13118 Wrong results with LOWER and UPPER and subquery
+#
+SET @save_optimizer_switch=@@optimizer_switch;
+SET optimizer_switch=_latin1'derived_merge=on';
+CREATE TABLE t1 AS SELECT REPEAT('a', 10) AS t LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `t` varchar(10) NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('abcdefghi'),('ABCDEFGHI');
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT LOWER(t) t2 FROM t1) sub;
+c2
+abcdefghi-abcdefghi
+abcdefghi-abcdefghi
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT UPPER(t) t2 FROM t1) sub;
+c2
+ABCDEFGHI-ABCDEFGHI
+ABCDEFGHI-ABCDEFGHI
+DROP TABLE t1;
+SET optimizer_switch=@save_optimizer_switch;
 #
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/ctype_ucs.result b/mysql-test/r/ctype_ucs.result
index 59d88414cab..58fd308c7e2 100644
--- a/mysql-test/r/ctype_ucs.result
+++ b/mysql-test/r/ctype_ucs.result
@@ -5629,6 +5629,29 @@ c2
 YWJjZGVmZ2hp-YWJjZGVmZ2hp
 DROP TABLE t1;
 SET optimizer_switch=@save_optimizer_switch;
+SET NAMES utf8, character_set_connection=ucs2;
+#
+# MDEV-13118 Wrong results with LOWER and UPPER and subquery
+#
+SET @save_optimizer_switch=@@optimizer_switch;
+SET optimizer_switch=_latin1'derived_merge=on';
+CREATE TABLE t1 AS SELECT REPEAT('a', 10) AS t LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `t` varchar(10) CHARACTER SET ucs2 NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('abcdefghi'),('ABCDEFGHI');
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT LOWER(t) t2 FROM t1) sub;
+c2
+abcdefghi-abcdefghi
+abcdefghi-abcdefghi
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT UPPER(t) t2 FROM t1) sub;
+c2
+ABCDEFGHI-ABCDEFGHI
+ABCDEFGHI-ABCDEFGHI
+DROP TABLE t1;
+SET optimizer_switch=@save_optimizer_switch;
 #
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/ctype_ujis.result b/mysql-test/r/ctype_ujis.result
index 413ab4efe31..d4589f62ae5 100644
--- a/mysql-test/r/ctype_ujis.result
+++ b/mysql-test/r/ctype_ujis.result
@@ -25942,6 +25942,29 @@ HEX(a)	CHAR_LENGTH(a)
 DROP TABLE t1;
 SELECT _ujis 0x8EA0;
 ERROR HY000: Invalid ujis character string: '8EA0'
+SET NAMES ujis;
+#
+# MDEV-13118 Wrong results with LOWER and UPPER and subquery
+#
+SET @save_optimizer_switch=@@optimizer_switch;
+SET optimizer_switch=_latin1'derived_merge=on';
+CREATE TABLE t1 AS SELECT REPEAT('a', 10) AS t LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `t` varchar(10) CHARACTER SET ujis NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('abcdefghi'),('ABCDEFGHI');
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT LOWER(t) t2 FROM t1) sub;
+c2
+abcdefghi-abcdefghi
+abcdefghi-abcdefghi
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT UPPER(t) t2 FROM t1) sub;
+c2
+ABCDEFGHI-ABCDEFGHI
+ABCDEFGHI-ABCDEFGHI
+DROP TABLE t1;
+SET optimizer_switch=@save_optimizer_switch;
 #
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/ctype_utf16.result b/mysql-test/r/ctype_utf16.result
index 958ade902df..e182432faec 100644
--- a/mysql-test/r/ctype_utf16.result
+++ b/mysql-test/r/ctype_utf16.result
@@ -2134,6 +2134,29 @@ EXECUTE stmt USING @arg00;
 CONCAT(_utf16'a' COLLATE utf16_unicode_ci, ?)
 aÿ
 DEALLOCATE PREPARE stmt;
+SET NAMES utf8, character_set_connection=utf16;
+#
+# MDEV-13118 Wrong results with LOWER and UPPER and subquery
+#
+SET @save_optimizer_switch=@@optimizer_switch;
+SET optimizer_switch=_latin1'derived_merge=on';
+CREATE TABLE t1 AS SELECT REPEAT('a', 10) AS t LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `t` varchar(10) CHARACTER SET utf16 NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('abcdefghi'),('ABCDEFGHI');
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT LOWER(t) t2 FROM t1) sub;
+c2
+abcdefghi-abcdefghi
+abcdefghi-abcdefghi
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT UPPER(t) t2 FROM t1) sub;
+c2
+ABCDEFGHI-ABCDEFGHI
+ABCDEFGHI-ABCDEFGHI
+DROP TABLE t1;
+SET optimizer_switch=@save_optimizer_switch;
 #
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/ctype_utf16le.result b/mysql-test/r/ctype_utf16le.result
index 8098b0d1666..f6a4d351ad4 100644
--- a/mysql-test/r/ctype_utf16le.result
+++ b/mysql-test/r/ctype_utf16le.result
@@ -2319,3 +2319,32 @@ DFFFFFDFFFFF9CFFFF9DFFFF9EFFFF
 #
 # End of 5.6 tests
 #
+#
+# Start of 10.0 tests
+#
+SET NAMES utf8, character_set_connection=utf16le;
+#
+# MDEV-13118 Wrong results with LOWER and UPPER and subquery
+#
+SET @save_optimizer_switch=@@optimizer_switch;
+SET optimizer_switch=_latin1'derived_merge=on';
+CREATE TABLE t1 AS SELECT REPEAT('a', 10) AS t LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `t` varchar(10) CHARACTER SET utf16le NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('abcdefghi'),('ABCDEFGHI');
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT LOWER(t) t2 FROM t1) sub;
+c2
+abcdefghi-abcdefghi
+abcdefghi-abcdefghi
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT UPPER(t) t2 FROM t1) sub;
+c2
+ABCDEFGHI-ABCDEFGHI
+ABCDEFGHI-ABCDEFGHI
+DROP TABLE t1;
+SET optimizer_switch=@save_optimizer_switch;
+#
+# Start of 10.0 tests
+#
diff --git a/mysql-test/r/ctype_utf32.result b/mysql-test/r/ctype_utf32.result
index 1f3e519a525..91277cd5108 100644
--- a/mysql-test/r/ctype_utf32.result
+++ b/mysql-test/r/ctype_utf32.result
@@ -2231,6 +2231,29 @@ EXECUTE stmt USING @arg00;
 CONCAT(_utf32'a' COLLATE utf32_unicode_ci, ?)
 aÿ
 DEALLOCATE PREPARE stmt;
+SET NAMEs utf8, character_set_connection=utf32;
+#
+# MDEV-13118 Wrong results with LOWER and UPPER and subquery
+#
+SET @save_optimizer_switch=@@optimizer_switch;
+SET optimizer_switch=_latin1'derived_merge=on';
+CREATE TABLE t1 AS SELECT REPEAT('a', 10) AS t LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `t` varchar(10) CHARACTER SET utf32 NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('abcdefghi'),('ABCDEFGHI');
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT LOWER(t) t2 FROM t1) sub;
+c2
+abcdefghi-abcdefghi
+abcdefghi-abcdefghi
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT UPPER(t) t2 FROM t1) sub;
+c2
+ABCDEFGHI-ABCDEFGHI
+ABCDEFGHI-ABCDEFGHI
+DROP TABLE t1;
+SET optimizer_switch=@save_optimizer_switch;
 #
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/ctype_utf8.result b/mysql-test/r/ctype_utf8.result
index 7a3ff3732c3..6a4001597c4 100644
--- a/mysql-test/r/ctype_utf8.result
+++ b/mysql-test/r/ctype_utf8.result
@@ -10127,6 +10127,29 @@ SELECT * FROM v1;
 c
 ß
 DROP VIEW v1;
+SET NAMES utf8;
+#
+# MDEV-13118 Wrong results with LOWER and UPPER and subquery
+#
+SET @save_optimizer_switch=@@optimizer_switch;
+SET optimizer_switch=_latin1'derived_merge=on';
+CREATE TABLE t1 AS SELECT REPEAT('a', 10) AS t LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `t` varchar(10) CHARACTER SET utf8 NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('abcdefghi'),('ABCDEFGHI');
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT LOWER(t) t2 FROM t1) sub;
+c2
+abcdefghi-abcdefghi
+abcdefghi-abcdefghi
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT UPPER(t) t2 FROM t1) sub;
+c2
+ABCDEFGHI-ABCDEFGHI
+ABCDEFGHI-ABCDEFGHI
+DROP TABLE t1;
+SET optimizer_switch=@save_optimizer_switch;
 #
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/ctype_utf8mb4.result b/mysql-test/r/ctype_utf8mb4.result
index f31b7a6781e..4d91c42cf51 100644
--- a/mysql-test/r/ctype_utf8mb4.result
+++ b/mysql-test/r/ctype_utf8mb4.result
@@ -3427,6 +3427,29 @@ t1	CREATE TABLE `t1` (
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1
 DROP TABLE t1;
 SET NAMES default;
+SET NAMES utf8mb4;
+#
+# MDEV-13118 Wrong results with LOWER and UPPER and subquery
+#
+SET @save_optimizer_switch=@@optimizer_switch;
+SET optimizer_switch=_latin1'derived_merge=on';
+CREATE TABLE t1 AS SELECT REPEAT('a', 10) AS t LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `t` varchar(10) CHARACTER SET utf8mb4 NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('abcdefghi'),('ABCDEFGHI');
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT LOWER(t) t2 FROM t1) sub;
+c2
+abcdefghi-abcdefghi
+abcdefghi-abcdefghi
+SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT UPPER(t) t2 FROM t1) sub;
+c2
+ABCDEFGHI-ABCDEFGHI
+ABCDEFGHI-ABCDEFGHI
+DROP TABLE t1;
+SET optimizer_switch=@save_optimizer_switch;
 # End of 10.0 tests
 #
 # End of tests
diff --git a/mysql-test/t/ctype_binary.test b/mysql-test/t/ctype_binary.test
index 3d3f90b444b..801e1328892 100644
--- a/mysql-test/t/ctype_binary.test
+++ b/mysql-test/t/ctype_binary.test
@@ -24,6 +24,9 @@ SET NAMES binary;
 --echo #
 SELECT _binary 0x7E, _binary X'7E', _binary B'01111110';
 
+SET NAMES utf8, character_set_connection=binary;
+--source include/ctype_mdev13118.inc
+
 --echo #
 --echo # End of 10.0 tests
 --echo #
diff --git a/mysql-test/t/ctype_eucjpms.test b/mysql-test/t/ctype_eucjpms.test
index 49ca81850ed..590f2ca9950 100644
--- a/mysql-test/t/ctype_eucjpms.test
+++ b/mysql-test/t/ctype_eucjpms.test
@@ -536,6 +536,8 @@ DROP TABLE t1;
 --error ER_INVALID_CHARACTER_STRING
 SELECT _eucjpms 0x8EA0;
 
+SET NAMES eucjpms;
+--source include/ctype_mdev13118.inc
 
 --echo #
 --echo # End of 10.0 tests
diff --git a/mysql-test/t/ctype_euckr.test b/mysql-test/t/ctype_euckr.test
index 155b8ebed00..d81ba2a8bdf 100644
--- a/mysql-test/t/ctype_euckr.test
+++ b/mysql-test/t/ctype_euckr.test
@@ -197,3 +197,14 @@ set collation_connection=euckr_bin;
 --echo # End of 5.6 tests
 --echo #
 
+
+--echo #
+--echo # Start of 10.0 tests
+--echo #
+
+SET NAMES utf8, character_set_connection=euckr;
+--source include/ctype_mdev13118.inc
+
+--echo #
+--echo # End of 10.0 tests
+--echo #
diff --git a/mysql-test/t/ctype_gbk.test b/mysql-test/t/ctype_gbk.test
index 454377d98a7..bcdcc09b473 100644
--- a/mysql-test/t/ctype_gbk.test
+++ b/mysql-test/t/ctype_gbk.test
@@ -199,6 +199,9 @@ let $ctype_unescape_combinations=selected;
 SET NAMES gbk;
 --source include/ctype_E05C.inc
 
+SET NAMES utf8, character_set_connection=gbk;
+--source include/ctype_mdev13118.inc
+
 --echo #
 --echo # MDEV-9886 Illegal mix of collations with a view comparing a field to a binary constant
 --echo #
diff --git a/mysql-test/t/ctype_latin1.test b/mysql-test/t/ctype_latin1.test
index 8a188b71e24..da4d76ee2cf 100644
--- a/mysql-test/t/ctype_latin1.test
+++ b/mysql-test/t/ctype_latin1.test
@@ -260,6 +260,9 @@ SELECT * FROM v1;
 DROP VIEW v1;
 DROP TABLE t1;
 
+SET NAMES latin1;
+--source include/ctype_mdev13118.inc
+
 
 --echo #
 --echo # End of 10.0 tests
diff --git a/mysql-test/t/ctype_ucs.test b/mysql-test/t/ctype_ucs.test
index d78977a3d1c..050e199e5b7 100644
--- a/mysql-test/t/ctype_ucs.test
+++ b/mysql-test/t/ctype_ucs.test
@@ -984,6 +984,10 @@ DROP TABLE t1;
 SET optimizer_switch=@save_optimizer_switch;
 
 
+SET NAMES utf8, character_set_connection=ucs2;
+--source include/ctype_mdev13118.inc
+
+
 --echo #
 --echo # End of 10.0 tests
 --echo #
diff --git a/mysql-test/t/ctype_ujis.test b/mysql-test/t/ctype_ujis.test
index 48dc0e63058..8d55819acf7 100644
--- a/mysql-test/t/ctype_ujis.test
+++ b/mysql-test/t/ctype_ujis.test
@@ -1366,6 +1366,10 @@ DROP TABLE t1;
 SELECT _ujis 0x8EA0;
 
 
+SET NAMES ujis;
+--source include/ctype_mdev13118.inc
+
+
 --echo #
 --echo # End of 10.0 tests
 --echo #
diff --git a/mysql-test/t/ctype_utf16.test b/mysql-test/t/ctype_utf16.test
index cda85239512..b562b15eeeb 100644
--- a/mysql-test/t/ctype_utf16.test
+++ b/mysql-test/t/ctype_utf16.test
@@ -866,6 +866,11 @@ SET @arg00=_binary 0x00FF;
 EXECUTE stmt USING @arg00;
 DEALLOCATE PREPARE stmt;
 
+
+SET NAMES utf8, character_set_connection=utf16;
+--source include/ctype_mdev13118.inc
+
+
 --echo #
 --echo # End of 10.0 tests
 --echo #
diff --git a/mysql-test/t/ctype_utf16le.test b/mysql-test/t/ctype_utf16le.test
index a8326900847..f5b89aa9dc4 100644
--- a/mysql-test/t/ctype_utf16le.test
+++ b/mysql-test/t/ctype_utf16le.test
@@ -744,3 +744,16 @@ SET NAMES utf8, collation_connection=utf16le_bin;
 --echo #
 --echo # End of 5.6 tests
 --echo #
+
+
+--echo #
+--echo # Start of 10.0 tests
+--echo #
+
+
+SET NAMES utf8, character_set_connection=utf16le;
+--source include/ctype_mdev13118.inc
+
+--echo #
+--echo # Start of 10.0 tests
+--echo #
diff --git a/mysql-test/t/ctype_utf32.test b/mysql-test/t/ctype_utf32.test
index c747abb8b08..1a4c9fd28ee 100644
--- a/mysql-test/t/ctype_utf32.test
+++ b/mysql-test/t/ctype_utf32.test
@@ -979,6 +979,14 @@ SET @arg00=_binary 0x00FF;
 EXECUTE stmt USING @arg00;
 DEALLOCATE PREPARE stmt;
 
+
+#
+# MDEV-13118 Wrong results with LOWER and UPPER and subquery
+#
+SET NAMEs utf8, character_set_connection=utf32;
+--source include/ctype_mdev13118.inc
+
+
 --echo #
 --echo # End of 10.0 tests
 --echo #
diff --git a/mysql-test/t/ctype_utf8.test b/mysql-test/t/ctype_utf8.test
index 2524daffa9e..11dfef67765 100644
--- a/mysql-test/t/ctype_utf8.test
+++ b/mysql-test/t/ctype_utf8.test
@@ -1880,6 +1880,13 @@ SELECT * FROM v1;
 DROP VIEW v1;
 
 
+#
+# MDEV-13118 Wrong results with LOWER and UPPER and subquery
+#
+SET NAMES utf8;
+--source include/ctype_mdev13118.inc
+
+
 --echo #
 --echo # End of 10.0 tests
 --echo #
diff --git a/mysql-test/t/ctype_utf8mb4.test b/mysql-test/t/ctype_utf8mb4.test
index f68cfd57e2a..89007546492 100644
--- a/mysql-test/t/ctype_utf8mb4.test
+++ b/mysql-test/t/ctype_utf8mb4.test
@@ -1913,6 +1913,14 @@ DROP TABLE t1;
 
 SET NAMES default;
 
+
+#
+# MDEV-13118 Wrong results with LOWER and UPPER and subquery
+#
+SET NAMES utf8mb4;
+--source include/ctype_mdev13118.inc
+
+
 --echo # End of 10.0 tests
 
 --echo #
diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc
index e0bd3cc2195..160e740a9f7 100644
--- a/sql/item_strfunc.cc
+++ b/sql/item_strfunc.cc
@@ -1566,32 +1566,27 @@ String *Item_str_conv::val_str(String *str)
 {
   DBUG_ASSERT(fixed == 1);
   String *res;
-  if (!(res=args[0]->val_str(str)))
-  {
-    null_value=1; /* purecov: inspected */
-    return 0; /* purecov: inspected */
-  }
-  null_value=0;
+  uint alloced_length, len;
+
+  if ((null_value= (!(res= args[0]->val_str(&tmp_value)) ||
+                    str->alloc((alloced_length= res->length() * multiply)))))
+    return 0;
+
   if (multiply == 1)
   {
-    uint len;
-    res= copy_if_not_alloced(&tmp_value, res, res->length());
-    len= converter(collation.collation, (char*) res->ptr(), res->length(),
-                                        (char*) res->ptr(), res->length());
-    DBUG_ASSERT(len <= res->length());
-    res->length(len);
+    str->copy(*res); // Should not fail (it was alloced above)
+    len= converter(collation.collation, (char*) str->ptr(), str->length(),
+                                        (char*) str->ptr(), alloced_length);
   }
   else
   {
-    uint len= res->length() * multiply;
-    tmp_value.alloc(len);
-    tmp_value.set_charset(collation.collation);
     len= converter(collation.collation, (char*) res->ptr(), res->length(),
-                                        (char*) tmp_value.ptr(), len);
-    tmp_value.length(len);
-    res= &tmp_value;
+                                        (char*) str->ptr(), alloced_length);
+    str->set_charset(collation.collation);
   }
-  return res;
+  DBUG_ASSERT(len <= alloced_length);
+  str->length(len);
+  return str;
 }
 
 
@@ -1783,7 +1778,7 @@ String *Item_func_substr_index::val_str(String *str)
   DBUG_ASSERT(fixed == 1);
   char buff[MAX_FIELD_WIDTH];
   String tmp(buff,sizeof(buff),system_charset_info);
-  String *res= args[0]->val_str(str);
+  String *res= args[0]->val_str(&tmp_value);
   String *delimiter= args[1]->val_str(&tmp);
   int32 count= (int32) args[2]->val_int();
   uint offset;
@@ -1832,20 +1827,31 @@ String *Item_func_substr_index::val_str(String *str)
       if (pass == 0) /* count<0 */
       {
         c+=n+1;
-        if (c<=0) return res; /* not found, return original string */
+        if (c<=0)
+        {
+          str->copy(res->ptr(), res->length(), collation.collation);
+          return str; // not found, return the original string
+        }
         ptr=res->ptr();
       }
       else
       {
-        if (c) return res; /* Not found, return original string */
+        if (c)
+        {
+          str->copy(res->ptr(), res->length(), collation.collation);
+          return str; // not found, return the original string
+        }
         if (count>0) /* return left part */
         {
-	  tmp_value.set(*res,0,(ulong) (ptr-res->ptr()));
+          str->copy(res->ptr(), (uint32) (ptr-res->ptr()), collation.collation);
+          return str;
         }
         else /* return right part */
         {
-	  ptr+= delimiter_length;
-	  tmp_value.set(*res,(ulong) (ptr-res->ptr()), (ulong) (strend-ptr));
+          ptr+= delimiter_length;
+          str->copy(res->ptr() + (ptr-res->ptr()), (uint32) (strend - ptr),
+                    collation.collation);
+          return str;
         }
       }
     }
@@ -1857,13 +1863,16 @@ String *Item_func_substr_index::val_str(String *str)
     {					// start counting from the beginning
       for (offset=0; ; offset+= delimiter_length)
       {
-	if ((int) (offset= res->strstr(*delimiter, offset)) < 0)
-	  return res;			// Didn't find, return org string
-	if (!--count)
-	{
-	  tmp_value.set(*res,0,offset);
-	  break;
-	}
+        if ((int) (offset= res->strstr(*delimiter, offset)) < 0)
+        {
+          str->copy(res->ptr(), res->length(), collation.collation);
+          return str; // not found, return the original string
+        }
+        if (!--count)
+        {
+          str->copy(res->ptr(), offset, collation.collation);
+          return str;
+        }
       }
     }
     else
@@ -1878,30 +1887,32 @@ String *Item_func_substr_index::val_str(String *str)
           address space less than where the found substring is located
           in res
         */
-	if ((int) (offset= res->strrstr(*delimiter, offset)) < 0)
-	  return res;			// Didn't find, return org string
+        if ((int) (offset= res->strrstr(*delimiter, offset)) < 0)
+        {
+          str->copy(res->ptr(), res->length(), collation.collation);
+          return str; // not found, return the original string
+        }
         /*
           At this point, we've searched for the substring
           the number of times as supplied by the index value
         */
-	if (!++count)
-	{
-	  offset+= delimiter_length;
-	  tmp_value.set(*res,offset,res->length()- offset);
-	  break;
-	}
+        if (!++count)
+        {
+          offset+= delimiter_length;
+          str->copy(res->ptr() + offset, res->length() - offset,
+                    collation.collation);
+          return str;
+        }
       }
       if (count)
-        return res;                     // Didn't find, return org string
+      {
+        str->copy(res->ptr(), res->length(), collation.collation);
+        return str; // not found, return the original string
+      }
     }
   }
-  /*
-    We always mark tmp_value as const so that if val_str() is called again
-    on this object, we don't disrupt the contents of tmp_value when it was
-    derived from another String.
-  */
-  tmp_value.mark_as_const();
-  return (&tmp_value);
+  DBUG_ASSERT(0);
+  return NULL;
 }
 
 /*

From e2ac4098ed507957ba6dac9b408d31054a8e6b6d Mon Sep 17 00:00:00 2001
From: Alexander Barkov <bar@mariadb.com>
Date: Thu, 19 Jul 2018 13:02:14 +0400
Subject: [PATCH 040/164] Simplify caseup() and casedn() in charsets

After the MDEV-13118 fix there's no code in the server that
wants caseup/casedn to change the argument in place for simple
charsets.  Let's remove this logic and always return the result in a
new string for all charsets, both simple and complex.

1. Removing the optimization that *some* character sets used in casedn()
  and caseup(), which allowed (and required) to change the case in-place,
  overwriting the string passed as the "src" argument.
  Now all CHARSET_INFO's work in the same way:
  non of them change the source string in-place, all of them now convert
  case from the source string to the destination string, leaving
  the source string untouched.

2. Adding "const" qualifier to the "char *src" parameter
   to caseup() and casedn().

3. Removing duplicate implementations in ctype-mb.c.
  Now both caseup() and casedn() implementations for all CJK character sets
  use internally the same function my_casefold_mb()
  (the former my_casefold_mb_varlen()).

4. Removing the "unused" attribute from parameters of some my_case{up|dn}_xxx()
   implementations, as the affected parameters are now *used* in the code.
   Previously these parameters were used only in DBUG_ASSERT().
---
 include/m_ctype.h      |  28 ++++++------
 sql/item_strfunc.cc    |  15 ++-----
 strings/ctype-bin.c    |   8 ++--
 strings/ctype-euc_kr.c |   4 +-
 strings/ctype-mb.c     | 100 ++++++-----------------------------------
 strings/ctype-simple.c |  22 +++++----
 strings/ctype-ucs2.c   |  78 +++++++++++++++++---------------
 strings/ctype-ujis.c   |   8 ++--
 strings/ctype-utf8.c   |  20 +++++----
 9 files changed, 106 insertions(+), 177 deletions(-)

diff --git a/include/m_ctype.h b/include/m_ctype.h
index 8d9838fdde2..eb2d760359b 100644
--- a/include/m_ctype.h
+++ b/include/m_ctype.h
@@ -365,7 +365,7 @@ typedef int (*my_charset_conv_mb_wc)(CHARSET_INFO *, my_wc_t *,
 typedef int (*my_charset_conv_wc_mb)(CHARSET_INFO *, my_wc_t,
                                      uchar *, uchar *);
 typedef size_t (*my_charset_conv_case)(CHARSET_INFO *,
-                                       char *, size_t, char *, size_t);
+                                       const char *, size_t, char *, size_t);
 
 
 /* See strings/CHARSET_INFO.txt about information on this structure  */
@@ -565,9 +565,11 @@ extern uint my_instr_simple(CHARSET_INFO *,
 /* Functions for 8bit */
 extern size_t my_caseup_str_8bit(CHARSET_INFO *, char *);
 extern size_t my_casedn_str_8bit(CHARSET_INFO *, char *);
-extern size_t my_caseup_8bit(CHARSET_INFO *, char *src, size_t srclen,
+extern size_t my_caseup_8bit(CHARSET_INFO *,
+                             const char *src, size_t srclen,
                              char *dst, size_t dstlen);
-extern size_t my_casedn_8bit(CHARSET_INFO *, char *src, size_t srclen,
+extern size_t my_casedn_8bit(CHARSET_INFO *,
+                             const char *src, size_t srclen,
                              char *dst, size_t dstlen);
 
 extern int my_strcasecmp_8bit(CHARSET_INFO * cs, const char *, const char *);
@@ -658,17 +660,17 @@ uint my_mbcharlen_8bit(CHARSET_INFO *, uint c);
 /* Functions for multibyte charsets */
 extern size_t my_caseup_str_mb(CHARSET_INFO *, char *);
 extern size_t my_casedn_str_mb(CHARSET_INFO *, char *);
-extern size_t my_caseup_mb(CHARSET_INFO *, char *src, size_t srclen,
-                                         char *dst, size_t dstlen);
-extern size_t my_casedn_mb(CHARSET_INFO *, char *src, size_t srclen,
-                                         char *dst, size_t dstlen);
-extern size_t my_caseup_mb_varlen(CHARSET_INFO *, char *src, size_t srclen,
-                                  char *dst, size_t dstlen);
-extern size_t my_casedn_mb_varlen(CHARSET_INFO *, char *src, size_t srclen,
-                                  char *dst, size_t dstlen);
-extern size_t my_caseup_ujis(CHARSET_INFO *, char *src, size_t srclen,
+extern size_t my_caseup_mb(CHARSET_INFO *,
+                           const char *src, size_t srclen,
+                           char *dst, size_t dstlen);
+extern size_t my_casedn_mb(CHARSET_INFO *,
+                           const char *src, size_t srclen,
+                           char *dst, size_t dstlen);
+extern size_t my_caseup_ujis(CHARSET_INFO *,
+                             const char *src, size_t srclen,
                              char *dst, size_t dstlen);
-extern size_t my_casedn_ujis(CHARSET_INFO *, char *src, size_t srclen,
+extern size_t my_casedn_ujis(CHARSET_INFO *,
+                             const char *src, size_t srclen,
                              char *dst, size_t dstlen);
 extern int my_strcasecmp_mb(CHARSET_INFO * cs,const char *, const char *);
 
diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc
index 160e740a9f7..ba9f14fb63f 100644
--- a/sql/item_strfunc.cc
+++ b/sql/item_strfunc.cc
@@ -1572,19 +1572,10 @@ String *Item_str_conv::val_str(String *str)
                     str->alloc((alloced_length= res->length() * multiply)))))
     return 0;
 
-  if (multiply == 1)
-  {
-    str->copy(*res); // Should not fail (it was alloced above)
-    len= converter(collation.collation, (char*) str->ptr(), str->length(),
-                                        (char*) str->ptr(), alloced_length);
-  }
-  else
-  {
-    len= converter(collation.collation, (char*) res->ptr(), res->length(),
-                                        (char*) str->ptr(), alloced_length);
-    str->set_charset(collation.collation);
-  }
+  len= converter(collation.collation, (char*) res->ptr(), res->length(),
+                                      (char*) str->ptr(), alloced_length);
   DBUG_ASSERT(len <= alloced_length);
+  str->set_charset(collation.collation);
   str->length(len);
   return str;
 }
diff --git a/strings/ctype-bin.c b/strings/ctype-bin.c
index 2e699db0bd3..53a0e850b54 100644
--- a/strings/ctype-bin.c
+++ b/strings/ctype-bin.c
@@ -220,11 +220,11 @@ static size_t my_case_str_bin(CHARSET_INFO *cs __attribute__((unused)),
 
 
 static size_t my_case_bin(CHARSET_INFO *cs __attribute__((unused)),
-                          char *src __attribute__((unused)),
-                          size_t srclen,
-                          char *dst __attribute__((unused)),
-                          size_t dstlen __attribute__((unused)))
+                          const char *src, size_t srclen,
+                          char *dst, size_t dstlen)
 {
+  DBUG_ASSERT(srclen <= dstlen);
+  memcpy(dst, src, srclen);
   return srclen;
 }
 
diff --git a/strings/ctype-euc_kr.c b/strings/ctype-euc_kr.c
index b7065369258..f35c3f07c79 100644
--- a/strings/ctype-euc_kr.c
+++ b/strings/ctype-euc_kr.c
@@ -9994,8 +9994,8 @@ static MY_CHARSET_HANDLER my_charset_handler=
   my_mb_ctype_mb,
   my_caseup_str_mb,
   my_casedn_str_mb,
-  my_caseup_mb_varlen, /* UPPER() can reduce length: Turkish DOTLESS i -> I */
-  my_casedn_mb,        /* LOWER() does not change length, use simple version*/
+  my_caseup_mb,        /* UPPER() can reduce length: Turkish DOTLESS i -> I */
+  my_casedn_mb,        /* LOWER() does not change length */
   my_snprintf_8bit,
   my_long10_to_str_8bit,
   my_longlong10_to_str_8bit,
diff --git a/strings/ctype-mb.c b/strings/ctype-mb.c
index 02a9a91ca6a..318e581792a 100644
--- a/strings/ctype-mb.c
+++ b/strings/ctype-mb.c
@@ -71,81 +71,8 @@ get_case_info_for_ch(const CHARSET_INFO *cs, uint page, uint offs)
 
 
 /*
-  For character sets which don't change octet length in case conversion.
-*/
-size_t my_caseup_mb(CHARSET_INFO * cs, char *src, size_t srclen,
-                    char *dst __attribute__((unused)),
-                    size_t dstlen __attribute__((unused)))
-{
-  register uint32 l;
-  register char *srcend= src + srclen;
-  register const uchar *map= cs->to_upper;
-
-  DBUG_ASSERT(cs->caseup_multiply == 1);
-  DBUG_ASSERT(src == dst && srclen == dstlen);
-  DBUG_ASSERT(cs->mbmaxlen == 2);
-  
-  while (src < srcend)
-  {
-    if ((l=my_ismbchar(cs, src, srcend)))
-    {
-      MY_UNICASE_CHARACTER *ch;
-      if ((ch= get_case_info_for_ch(cs, (uchar) src[0], (uchar) src[1])))
-      {
-        *src++= ch->toupper >> 8;
-        *src++= ch->toupper & 0xFF;
-      }
-      else
-        src+= l;
-    }
-    else 
-    {
-      *src=(char) map[(uchar) *src];
-      src++;
-    }
-  }
-  return srclen;
-}
-
-
-size_t my_casedn_mb(CHARSET_INFO * cs, char *src, size_t srclen,
-                    char *dst __attribute__((unused)),
-                    size_t dstlen __attribute__((unused)))
-{
-  register uint32 l;
-  register char *srcend= src + srclen;
-  register const uchar *map=cs->to_lower;
-
-  DBUG_ASSERT(cs->casedn_multiply == 1);
-  DBUG_ASSERT(src == dst && srclen == dstlen);  
-  DBUG_ASSERT(cs->mbmaxlen == 2);
-  
-  while (src < srcend)
-  {
-    if ((l= my_ismbchar(cs, src, srcend)))
-    {
-      MY_UNICASE_CHARACTER *ch;
-      if ((ch= get_case_info_for_ch(cs, (uchar) src[0], (uchar) src[1])))
-      {
-        *src++= ch->tolower >> 8;
-        *src++= ch->tolower & 0xFF;
-      }
-      else
-        src+= l;
-    }
-    else
-    {
-      *src= (char) map[(uchar)*src];
-      src++;
-    }
-  }
-  return srclen;
-}
-
-
-/*
-  Case folding functions for character set
-  where case conversion can change string octet length.
+  Case folding functions for CJK character set.
+  Case conversion can optionally reduce string octet length.
   For example, in EUCKR,
     _euckr 0xA9A5 == "LATIN LETTER DOTLESS I" (Turkish letter)
   is upper-cased to to
@@ -153,13 +80,14 @@ size_t my_casedn_mb(CHARSET_INFO * cs, char *src, size_t srclen,
   Length is reduced in this example from two bytes to one byte.
 */
 static size_t
-my_casefold_mb_varlen(CHARSET_INFO *cs,
-                      char *src, size_t srclen,
-                      char *dst, size_t dstlen __attribute__((unused)),
-                      const uchar *map,
-                      size_t is_upper)
+my_casefold_mb(CHARSET_INFO *cs,
+               const char *src, size_t srclen,
+               char *dst, size_t dstlen __attribute__((unused)),
+               const uchar *map,
+               size_t is_upper)
 {
-  char *srcend= src + srclen, *dst0= dst;
+  const char *srcend= src + srclen;
+  char *dst0= dst;
 
   DBUG_ASSERT(cs->mbmaxlen == 2);
 
@@ -193,22 +121,22 @@ my_casefold_mb_varlen(CHARSET_INFO *cs,
 
 
 size_t
-my_casedn_mb_varlen(CHARSET_INFO * cs, char *src, size_t srclen,
+my_casedn_mb(CHARSET_INFO * cs, const char *src, size_t srclen,
                     char *dst, size_t dstlen)
 {
   DBUG_ASSERT(dstlen >= srclen * cs->casedn_multiply); 
   DBUG_ASSERT(src != dst || cs->casedn_multiply == 1);
-  return my_casefold_mb_varlen(cs, src, srclen, dst, dstlen, cs->to_lower, 0);
+  return my_casefold_mb(cs, src, srclen, dst, dstlen, cs->to_lower, 0);
 }
 
 
 size_t
-my_caseup_mb_varlen(CHARSET_INFO * cs, char *src, size_t srclen,
-                    char *dst, size_t dstlen)
+my_caseup_mb(CHARSET_INFO * cs, const char *src, size_t srclen,
+             char *dst, size_t dstlen)
 {
   DBUG_ASSERT(dstlen >= srclen * cs->caseup_multiply);
   DBUG_ASSERT(src != dst || cs->caseup_multiply == 1);
-  return my_casefold_mb_varlen(cs, src, srclen, dst, dstlen, cs->to_upper, 1);
+  return my_casefold_mb(cs, src, srclen, dst, dstlen, cs->to_upper, 1);
 }
 
 
diff --git a/strings/ctype-simple.c b/strings/ctype-simple.c
index 7f13cef4474..61b14b84820 100644
--- a/strings/ctype-simple.c
+++ b/strings/ctype-simple.c
@@ -214,28 +214,26 @@ size_t my_casedn_str_8bit(CHARSET_INFO * cs,char *str)
 }
 
 
-size_t my_caseup_8bit(CHARSET_INFO * cs, char *src, size_t srclen,
-                      char *dst __attribute__((unused)),
-                      size_t dstlen __attribute__((unused)))
+size_t my_caseup_8bit(CHARSET_INFO * cs, const char *src, size_t srclen,
+                      char *dst, size_t dstlen)
 {
-  char *end= src + srclen;
+  const char *end= src + srclen;
   register const uchar *map= cs->to_upper;
-  DBUG_ASSERT(src == dst && srclen == dstlen);
+  DBUG_ASSERT(srclen <= dstlen);
   for ( ; src != end ; src++)
-    *src= (char) map[(uchar) *src];
+    *dst++= (char) map[(uchar) *src];
   return srclen;
 }
 
 
-size_t my_casedn_8bit(CHARSET_INFO * cs, char *src, size_t srclen,
-                      char *dst __attribute__((unused)),
-                      size_t dstlen __attribute__((unused)))
+size_t my_casedn_8bit(CHARSET_INFO * cs, const char *src, size_t srclen,
+                      char *dst, size_t dstlen)
 {
-  char *end= src + srclen;
+  const char *end= src + srclen;
   register const uchar *map=cs->to_lower;
-  DBUG_ASSERT(src == dst && srclen == dstlen);
+  DBUG_ASSERT(srclen <= dstlen);
   for ( ; src != end ; src++)
-    *src= (char) map[(uchar) *src];
+    *dst++= (char) map[(uchar) *src];
   return srclen;
 }
 
diff --git a/strings/ctype-ucs2.c b/strings/ctype-ucs2.c
index 318c248f742..724e69872de 100644
--- a/strings/ctype-ucs2.c
+++ b/strings/ctype-ucs2.c
@@ -1196,25 +1196,26 @@ my_tosort_utf16(MY_UNICASE_INFO *uni_plane, my_wc_t *wc)
 
 
 static size_t
-my_caseup_utf16(CHARSET_INFO *cs, char *src, size_t srclen,
-                char *dst __attribute__((unused)),
-                size_t dstlen __attribute__((unused)))
+my_caseup_utf16(CHARSET_INFO *cs, const char *src, size_t srclen,
+                char *dst, size_t dstlen)
 {
   my_wc_t wc;
   my_charset_conv_mb_wc mb_wc= cs->cset->mb_wc;
   my_charset_conv_wc_mb wc_mb= cs->cset->wc_mb;
   int res;
-  char *srcend= src + srclen;
+  const char *srcend= src + srclen;
+  char *dstend= dst + dstlen;
   MY_UNICASE_INFO *uni_plane= cs->caseinfo;
-  DBUG_ASSERT(src == dst && srclen == dstlen);
+  DBUG_ASSERT(srclen <= dstlen);
   
   while ((src < srcend) &&
          (res= mb_wc(cs, &wc, (uchar *) src, (uchar *) srcend)) > 0)
   {
     my_toupper_utf16(uni_plane, &wc);
-    if (res != wc_mb(cs, wc, (uchar *) src, (uchar *) srcend))
+    if (res != wc_mb(cs, wc, (uchar *) dst, (uchar *) dstend))
       break;
     src+= res;
+    dst+= res;
   }
   return srclen;
 }
@@ -1243,25 +1244,26 @@ my_hash_sort_utf16(CHARSET_INFO *cs, const uchar *s, size_t slen,
 
 
 static size_t
-my_casedn_utf16(CHARSET_INFO *cs, char *src, size_t srclen,
-                char *dst __attribute__((unused)),
-                size_t dstlen __attribute__((unused)))
+my_casedn_utf16(CHARSET_INFO *cs, const char *src, size_t srclen,
+                char *dst, size_t dstlen)
 {
   my_wc_t wc;
   my_charset_conv_mb_wc mb_wc= cs->cset->mb_wc;
   my_charset_conv_wc_mb wc_mb= cs->cset->wc_mb;
   int res;
-  char *srcend= src + srclen;
+  const char *srcend= src + srclen;
+  char *dstend= dst + dstlen;
   MY_UNICASE_INFO *uni_plane= cs->caseinfo;
-  DBUG_ASSERT(src == dst && srclen == dstlen);
+  DBUG_ASSERT(srclen <= dstlen);
 
   while ((src < srcend) &&
          (res= mb_wc(cs, &wc, (uchar *) src, (uchar *) srcend)) > 0)
   {
     my_tolower_utf16(uni_plane, &wc);
-    if (res != wc_mb(cs, wc, (uchar *) src, (uchar *) srcend))
+    if (res != wc_mb(cs, wc, (uchar *) dst, (uchar *) dstend))
       break;
     src+= res;
+    dst+= res;
   }
   return srclen;
 }
@@ -1987,23 +1989,24 @@ my_tosort_utf32(MY_UNICASE_INFO *uni_plane, my_wc_t *wc)
 
 
 static size_t
-my_caseup_utf32(CHARSET_INFO *cs, char *src, size_t srclen,
-                char *dst __attribute__((unused)),
-                size_t dstlen __attribute__((unused)))
+my_caseup_utf32(CHARSET_INFO *cs, const char *src, size_t srclen,
+                char *dst, size_t dstlen)
 {
   my_wc_t wc;
   int res;
-  char *srcend= src + srclen;
+  const char *srcend= src + srclen;
+  char *dstend= dst + dstlen;
   MY_UNICASE_INFO *uni_plane= cs->caseinfo;
-  DBUG_ASSERT(src == dst && srclen == dstlen);
+  DBUG_ASSERT(srclen <= dstlen);
   
   while ((src < srcend) &&
          (res= my_utf32_uni(cs, &wc, (uchar *)src, (uchar*) srcend)) > 0)
   {
     my_toupper_utf32(uni_plane, &wc);
-    if (res != my_uni_utf32(cs, wc, (uchar*) src, (uchar*) srcend))
+    if (res != my_uni_utf32(cs, wc, (uchar*) dst, (uchar*) dstend))
       break;
     src+= res;
+    dst+= res;
   }
   return srclen;
 }
@@ -2038,22 +2041,23 @@ my_hash_sort_utf32(CHARSET_INFO *cs, const uchar *s, size_t slen,
 
 
 static size_t
-my_casedn_utf32(CHARSET_INFO *cs, char *src, size_t srclen,
-                char *dst __attribute__((unused)),
-                size_t dstlen __attribute__((unused)))
+my_casedn_utf32(CHARSET_INFO *cs, const char *src, size_t srclen,
+                char *dst, size_t dstlen)
 {
   my_wc_t wc;
   int res;
-  char *srcend= src + srclen;
+  const char *srcend= src + srclen;
+  char *dstend= dst + dstlen;
   MY_UNICASE_INFO *uni_plane= cs->caseinfo;
-  DBUG_ASSERT(src == dst && srclen == dstlen);
+  DBUG_ASSERT(srclen <= dstlen);
 
   while ((res= my_utf32_uni(cs, &wc, (uchar*) src, (uchar*) srcend)) > 0)
   {
     my_tolower_utf32(uni_plane,&wc);
-    if (res != my_uni_utf32(cs, wc, (uchar*) src, (uchar*) srcend))
+    if (res != my_uni_utf32(cs, wc, (uchar*) dst, (uchar*) dstend))
       break;
     src+= res;
+    dst+= res;
   }
   return srclen;
 }
@@ -2950,23 +2954,24 @@ my_tosort_ucs2(MY_UNICASE_INFO *uni_plane, my_wc_t *wc)
     *wc= page[*wc & 0xFF].sort;
 }
 
-static size_t my_caseup_ucs2(CHARSET_INFO *cs, char *src, size_t srclen,
-                           char *dst __attribute__((unused)),
-                           size_t dstlen __attribute__((unused)))
+static size_t my_caseup_ucs2(CHARSET_INFO *cs, const char *src, size_t srclen,
+                           char *dst, size_t dstlen)
 {
   my_wc_t wc;
   int res;
-  char *srcend= src + srclen;
+  const char *srcend= src + srclen;
+  char *dstend= dst + dstlen;
   MY_UNICASE_INFO *uni_plane= cs->caseinfo;
-  DBUG_ASSERT(src == dst && srclen == dstlen);
+  DBUG_ASSERT(srclen <= dstlen);
   
   while ((src < srcend) &&
          (res= my_ucs2_uni(cs, &wc, (uchar *)src, (uchar*) srcend)) > 0)
   {
     my_toupper_ucs2(uni_plane, &wc);
-    if (res != my_uni_ucs2(cs, wc, (uchar*) src, (uchar*) srcend))
+    if (res != my_uni_ucs2(cs, wc, (uchar*) dst, (uchar*) dstend))
       break;
     src+= res;
+    dst+= res;
   }
   return srclen;
 }
@@ -2995,23 +3000,24 @@ static void my_hash_sort_ucs2(CHARSET_INFO *cs, const uchar *s, size_t slen,
 }
 
 
-static size_t my_casedn_ucs2(CHARSET_INFO *cs, char *src, size_t srclen,
-                           char *dst __attribute__((unused)),
-                           size_t dstlen __attribute__((unused)))
+static size_t my_casedn_ucs2(CHARSET_INFO *cs, const char *src, size_t srclen,
+                           char *dst, size_t dstlen)
 {
   my_wc_t wc;
   int res;
-  char *srcend= src + srclen;
+  const char *srcend= src + srclen;
+  char *dstend= dst + dstlen;
   MY_UNICASE_INFO *uni_plane= cs->caseinfo;
-  DBUG_ASSERT(src == dst && srclen == dstlen);
+  DBUG_ASSERT(srclen <= dstlen);
 
   while ((src < srcend) &&
          (res= my_ucs2_uni(cs, &wc, (uchar*) src, (uchar*) srcend)) > 0)
   {
     my_tolower_ucs2(uni_plane, &wc);
-    if (res != my_uni_ucs2(cs, wc, (uchar*) src, (uchar*) srcend))
+    if (res != my_uni_ucs2(cs, wc, (uchar*) dst, (uchar*) dstend))
       break;
     src+= res;
+    dst+= res;
   }
   return srclen;
 }
diff --git a/strings/ctype-ujis.c b/strings/ctype-ujis.c
index e7dbefe6c1d..25f8881d819 100644
--- a/strings/ctype-ujis.c
+++ b/strings/ctype-ujis.c
@@ -67179,12 +67179,12 @@ get_case_info_for_ch(CHARSET_INFO *cs, uint plane, uint page, uint offs)
 */
 static size_t
 my_casefold_ujis(CHARSET_INFO *cs,
-                 char *src, size_t srclen,
+                 const char *src, size_t srclen,
                  char *dst, size_t dstlen __attribute__((unused)),
                  const uchar * const map,
                  size_t is_upper)
 {
-  char *srcend= src + srclen, *dst0= dst;
+  const char *srcend= src + srclen, *dst0= dst;
 
   while (src < srcend)
   {
@@ -67226,7 +67226,7 @@ my_casefold_ujis(CHARSET_INFO *cs,
   LOWER()
 */
 size_t
-my_casedn_ujis(CHARSET_INFO * cs, char *src, size_t srclen,
+my_casedn_ujis(CHARSET_INFO * cs, const char *src, size_t srclen,
                char *dst, size_t dstlen)
 {
   DBUG_ASSERT(dstlen >= srclen * cs->casedn_multiply); 
@@ -67239,7 +67239,7 @@ my_casedn_ujis(CHARSET_INFO * cs, char *src, size_t srclen,
   UPPER()
 */
 size_t
-my_caseup_ujis(CHARSET_INFO * cs, char *src, size_t srclen,
+my_caseup_ujis(CHARSET_INFO * cs, const char *src, size_t srclen,
                char *dst, size_t dstlen)
 {
   DBUG_ASSERT(dstlen >= srclen * cs->caseup_multiply);
diff --git a/strings/ctype-utf8.c b/strings/ctype-utf8.c
index cd4e82a107c..7d7e61ce511 100644
--- a/strings/ctype-utf8.c
+++ b/strings/ctype-utf8.c
@@ -5065,12 +5065,13 @@ my_tosort_utf8mb3(MY_UNICASE_INFO *uni_plane, my_wc_t *wc)
     *wc= page[*wc & 0xFF].sort;
 }
 
-static size_t my_caseup_utf8(CHARSET_INFO *cs, char *src, size_t srclen,
+static size_t my_caseup_utf8(CHARSET_INFO *cs, const char *src, size_t srclen,
                              char *dst, size_t dstlen)
 {
   my_wc_t wc;
   int srcres, dstres;
-  char *srcend= src + srclen, *dstend= dst + dstlen, *dst0= dst;
+  const char *srcend= src + srclen;
+  char *dstend= dst + dstlen, *dst0= dst;
   MY_UNICASE_INFO *uni_plane= cs->caseinfo;
   DBUG_ASSERT(src != dst || cs->caseup_multiply == 1);
 
@@ -5136,12 +5137,13 @@ static size_t my_caseup_str_utf8(CHARSET_INFO *cs, char *src)
 }
 
 
-static size_t my_casedn_utf8(CHARSET_INFO *cs, char *src, size_t srclen,
+static size_t my_casedn_utf8(CHARSET_INFO *cs, const char *src, size_t srclen,
                              char *dst, size_t dstlen)
 {
   my_wc_t wc;
   int srcres, dstres;
-  char *srcend= src + srclen, *dstend= dst + dstlen, *dst0= dst;
+  const char *srcend= src + srclen;
+  char *dstend= dst + dstlen, *dst0= dst;
   MY_UNICASE_INFO *uni_plane= cs->caseinfo;
   DBUG_ASSERT(src != dst || cs->casedn_multiply == 1);
 
@@ -7579,12 +7581,13 @@ my_toupper_utf8mb4(MY_UNICASE_INFO *uni_plane, my_wc_t *wc)
 
 
 static size_t
-my_caseup_utf8mb4(CHARSET_INFO *cs, char *src, size_t srclen,
+my_caseup_utf8mb4(CHARSET_INFO *cs, const char *src, size_t srclen,
                   char *dst, size_t dstlen)
 {
   my_wc_t wc;
   int srcres, dstres;
-  char *srcend= src + srclen, *dstend= dst + dstlen, *dst0= dst;
+  const char *srcend= src + srclen;
+  char *dstend= dst + dstlen, *dst0= dst;
   MY_UNICASE_INFO *uni_plane= cs->caseinfo;
   DBUG_ASSERT(src != dst || cs->caseup_multiply == 1);
 
@@ -7666,12 +7669,13 @@ my_caseup_str_utf8mb4(CHARSET_INFO *cs, char *src)
 
 static size_t
 my_casedn_utf8mb4(CHARSET_INFO *cs,
-                  char *src, size_t srclen,
+                  const char *src, size_t srclen,
                   char *dst, size_t dstlen)
 {
   my_wc_t wc;
   int srcres, dstres;
-  char *srcend= src + srclen, *dstend= dst + dstlen, *dst0= dst;
+  const char *srcend= src + srclen;
+  char *dstend= dst + dstlen, *dst0= dst;
   MY_UNICASE_INFO *uni_plane= cs->caseinfo;
   DBUG_ASSERT(src != dst || cs->casedn_multiply == 1);
 

From 09f147659f36a30d7075885d88392c9a41244505 Mon Sep 17 00:00:00 2001
From: Sergei Petrunia <psergey@askmonty.org>
Date: Thu, 19 Jul 2018 12:07:07 +0300
Subject: [PATCH 041/164] MDEV-16777: galera.galera_gra_log fails with File
 ...GRA_*.log not found error

snprintf returns the number of bytes it wrote (or would have written) NOT
counting the \0 terminal character.
The buffer size it accepts as argument DOES COUNT the \0 character.
Pass the right parameter value.
---
 sql/wsrep_binlog.cc | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/sql/wsrep_binlog.cc b/sql/wsrep_binlog.cc
index 902190d4772..c7674cd0169 100644
--- a/sql/wsrep_binlog.cc
+++ b/sql/wsrep_binlog.cc
@@ -326,11 +326,16 @@ void wsrep_dump_rbr_buf(THD *thd, const void* rbr_buf, size_t buf_len)
     WSREP_ERROR("snprintf error: %d, skipping dump.", len);
     return;
   }
+  /*
+    len doesn't count the \0 end-of-string. Use len+1 below
+    to alloc and pass as an argument to snprintf.
+  */
 
   char *filename= (char *)malloc(len+1);
-  int len1= snprintf(filename, len, "%s/GRA_%ld_%lld.log",
+  int len1= snprintf(filename, len+1, "%s/GRA_%ld_%lld.log",
                     wsrep_data_home_dir, thd->thread_id,
                     (long long)wsrep_thd_trx_seqno(thd));
+
   if (len > len1)
   {
     WSREP_ERROR("RBR dump path truncated: %d, skipping dump.", len);
@@ -469,7 +474,10 @@ void wsrep_dump_rbr_buf_with_header(THD *thd, const void *rbr_buf,
   int len= snprintf(NULL, 0, "%s/GRA_%ld_%lld_v2.log",
                     wsrep_data_home_dir, thd->thread_id,
                     thd_trx_seqno);
-
+  /*
+    len doesn't count the \0 end-of-string. Use len+1 below
+    to alloc and pass as an argument to snprintf.
+  */
   char *filename;
   if (len < 0 || !(filename= (char*)malloc(len+1)))
   {
@@ -477,7 +485,7 @@ void wsrep_dump_rbr_buf_with_header(THD *thd, const void *rbr_buf,
     DBUG_VOID_RETURN;
   }
 
-  int len1= snprintf(filename, len, "%s/GRA_%ld_%lld_v2.log",
+  int len1= snprintf(filename, len+1, "%s/GRA_%ld_%lld_v2.log",
                      wsrep_data_home_dir, thd->thread_id,
                      thd_trx_seqno);
 

From 0b3e28a4cd3f8000eedb8ba190c0ea461544651a Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Thu, 12 Jul 2018 21:58:11 +0200
Subject: [PATCH 042/164] MDEV-8941 Compile on Solaris (SPARC) fails with
 errors in filamvct.cpp

remove unnecessary declaration
---
 storage/connect/filamvct.cpp | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/storage/connect/filamvct.cpp b/storage/connect/filamvct.cpp
index 244acfdc5c8..dd827d084fa 100755
--- a/storage/connect/filamvct.cpp
+++ b/storage/connect/filamvct.cpp
@@ -65,11 +65,6 @@
 extern int num_read, num_there;                          // Statistics
 static int num_write;
 
-#if defined(UNIX)
-// Add dummy strerror  (NGC)
-char *strerror(int num);
-#endif   // UNIX
-
 /***********************************************************************/
 /*  Header containing block info for not split VEC tables.             */
 /*  Block and last values can be calculated from NumRec and Nrec.      */

From bd5cf02bbe7bce029b0275be1b15d2108806d5e9 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Fri, 13 Jul 2018 16:54:47 +0200
Subject: [PATCH 043/164] MDEV-11741 handler::ha_reset(): Assertion
 `bitmap_is_set_all(&table->s->all_set)' failed or server crash in mi_reset or
 buffer overrun or unexpected ER_CANT_REMOVE_ALL_FIELDS

MEMORY table could be renamed into a non-extistent database.

rename() is documented to return ENOENT when the source file does not
exist OR when the target directory not exist. Nonexistent source .frm
file is ok (table can still exist in the engine), nonexistent target
directory is not.

Make my_rename to use ENOTDIR for the latter case. Make RENAME TABLE
issue an appropriate error ("unknown database" instead of "unknown table")
---
 mysql-test/r/rename.result | 4 ++++
 mysql-test/t/rename.test   | 7 +++++++
 mysys/my_rename.c          | 5 ++++-
 sql/sql_table.cc           | 2 ++
 4 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/mysql-test/r/rename.result b/mysql-test/r/rename.result
index c90aaf24e9b..f019527c951 100644
--- a/mysql-test/r/rename.result
+++ b/mysql-test/r/rename.result
@@ -133,3 +133,7 @@ select * from t2;
 a
 1
 drop table tmp,t2;
+create table t1 (a int) engine=memory;
+rename table t1 to non_existent.t2;
+ERROR 42000: Unknown database 'non_existent'
+drop table t1;
diff --git a/mysql-test/t/rename.test b/mysql-test/t/rename.test
index 67732d5b5b9..215ecbcbb18 100644
--- a/mysql-test/t/rename.test
+++ b/mysql-test/t/rename.test
@@ -141,3 +141,10 @@ select * from tmp;
 select * from t2;
 drop table tmp,t2;
 
+#
+# MDEV-11741 handler::ha_reset(): Assertion `bitmap_is_set_all(&table->s->all_set)' failed or server crash in mi_reset or buffer overrun or unexpected ER_CANT_REMOVE_ALL_FIELDS
+#
+create table t1 (a int) engine=memory;
+--error ER_BAD_DB_ERROR
+rename table t1 to non_existent.t2;
+drop table t1;
diff --git a/mysys/my_rename.c b/mysys/my_rename.c
index 09e7eafa980..17f693629a8 100644
--- a/mysys/my_rename.c
+++ b/mysys/my_rename.c
@@ -39,7 +39,10 @@ int my_rename(const char *from, const char *to, myf MyFlags)
   if (link(from, to) || unlink(from))
   {
 #endif
-    my_errno=errno;
+    if (errno == ENOENT && !access(from, F_OK))
+      my_errno= ENOTDIR;
+    else
+      my_errno= errno;
     error = -1;
     if (MyFlags & (MY_FAE+MY_WME))
       my_error(EE_LINK, MYF(ME_BELL+ME_WAITTANG),from,to,my_errno);
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index ee02b5fc7aa..9a2b4901d4e 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -5257,6 +5257,8 @@ mysql_rename_table(handlerton *base, const char *old_db,
   delete file;
   if (error == HA_ERR_WRONG_COMMAND)
     my_error(ER_NOT_SUPPORTED_YET, MYF(0), "ALTER TABLE");
+  else if (error ==  ENOTDIR)
+    my_error(ER_BAD_DB_ERROR, MYF(0), new_db);
   else if (error)
     my_error(ER_ERROR_ON_RENAME, MYF(0), from, to, error);
   else if (!(flags & FN_IS_TMP))

From 33eccb577657a127a877cea53ee4dc41cb4604c2 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Fri, 13 Jul 2018 21:37:22 +0200
Subject: [PATCH 044/164] MDEV-11790 WITHOUT_SERVER installs mysqld_safe_helper

Don't install server files if WITHOUT_SERVER is specified.
"Server files" are defined as files going into the MariaDB-Server RPM,
that is files in the components Server, ManPagesServer, Server_Scripts,
IniFiles, SuportFiles, and Readme.
---
 CMakeLists.txt                   |  6 ++++++
 cmake/install_macros.cmake       |  8 ++++++++
 cmake/mysql_add_executable.cmake |  3 +++
 scripts/CMakeLists.txt           |  2 ++
 sql/share/CMakeLists.txt         | 12 +++++++-----
 support-files/CMakeLists.txt     |  2 ++
 6 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 11874650d18..5a9ec265de0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -165,6 +165,12 @@ ENDIF()
 OPTION (WITH_UNIT_TESTS "Compile MySQL with unit tests" ON)
 MARK_AS_ADVANCED(CYBOZU BACKUP_TEST WITHOUT_SERVER DISABLE_SHARED)
 
+IF (WITHOUT_SERVER)
+  SET (SKIP_COMPONENTS "Server|IniFiles|SuportFiles|Readme")
+ELSE()
+  SET (SKIP_COMPONENTS "N-O-N-E")
+ENDIF()
+
 OPTION(NOT_FOR_DISTRIBUTION "Allow linking with GPLv2-incompatible system libraries. Only set it you never plan to distribute the resulting binaries" OFF)
 
 include(CheckCSourceCompiles)
diff --git a/cmake/install_macros.cmake b/cmake/install_macros.cmake
index 7da8edd22eb..ff4ba593415 100644
--- a/cmake/install_macros.cmake
+++ b/cmake/install_macros.cmake
@@ -115,6 +115,10 @@ FUNCTION(INSTALL_SCRIPT)
     SET(COMP)
   ENDIF()
 
+  IF (COMP MATCHES ${SKIP_COMPONENTS})
+    RETURN()
+  ENDIF()
+
   INSTALL(FILES 
     ${script}
     DESTINATION ${ARG_DESTINATION}
@@ -138,6 +142,10 @@ FUNCTION(INSTALL_DOCUMENTATION)
     SET(destination ${INSTALL_DOCDIR})
   ENDIF()
 
+  IF (ARG_COMPONENT MATCHES ${SKIP_COMPONENTS})
+    RETURN()
+  ENDIF()
+
   STRING(TOUPPER ${ARG_COMPONENT} COMPUP)
   IF(CPACK_COMPONENT_${COMPUP}_GROUP)
     SET(group ${CPACK_COMPONENT_${COMPUP}_GROUP})
diff --git a/cmake/mysql_add_executable.cmake b/cmake/mysql_add_executable.cmake
index 45575bdd536..c8a2e522d66 100644
--- a/cmake/mysql_add_executable.cmake
+++ b/cmake/mysql_add_executable.cmake
@@ -59,6 +59,9 @@ FUNCTION (MYSQL_ADD_EXECUTABLE)
     ELSE()
       SET(COMP COMPONENT Client)
     ENDIF()
+    IF (COMP MATCHES ${SKIP_COMPONENTS})
+      RETURN()
+    ENDIF()
     MYSQL_INSTALL_TARGETS(${target} DESTINATION ${ARG_DESTINATION} ${COMP})
   ENDIF()
 ENDFUNCTION()
diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt
index 40273b1a1c2..24b48e1920e 100644
--- a/scripts/CMakeLists.txt
+++ b/scripts/CMakeLists.txt
@@ -74,6 +74,7 @@ IF(UNIX)
 )
 ENDIF()
 
+IF (NOT WITHOUT_SERVER)
 INSTALL(FILES 
   ${CMAKE_CURRENT_SOURCE_DIR}/mysql_system_tables.sql 
   ${CMAKE_CURRENT_SOURCE_DIR}/mysql_system_tables_data.sql
@@ -83,6 +84,7 @@ INSTALL(FILES
   ${FIX_PRIVILEGES_SQL}
   DESTINATION ${INSTALL_MYSQLSHAREDIR} COMPONENT Server
 )
+ENDIF()
 
 # TCMalloc hacks
 IF(MALLOC_LIB)
diff --git a/sql/share/CMakeLists.txt b/sql/share/CMakeLists.txt
index e0d5fb6c1a7..4293c0b528c 100644
--- a/sql/share/CMakeLists.txt
+++ b/sql/share/CMakeLists.txt
@@ -44,12 +44,14 @@ SET(files
  errmsg-utf8.txt
 )
 
-FOREACH (dir ${dirs})
- INSTALL(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${dir} 
-  DESTINATION  ${INSTALL_MYSQLSHAREDIR} COMPONENT Server)
-ENDFOREACH()
 INSTALL(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/charsets DESTINATION ${INSTALL_MYSQLSHAREDIR}
   COMPONENT Common PATTERN "languages.html" EXCLUDE
 )
 
-INSTALL(FILES ${files} DESTINATION ${INSTALL_MYSQLSHAREDIR} COMPONENT Server)
+IF (NOT WITHOUT_SERVER)
+  FOREACH (dir ${dirs})
+   INSTALL(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${dir}
+    DESTINATION  ${INSTALL_MYSQLSHAREDIR} COMPONENT Server)
+  ENDFOREACH()
+  INSTALL(FILES ${files} DESTINATION ${INSTALL_MYSQLSHAREDIR} COMPONENT Server)
+ENDIF()
diff --git a/support-files/CMakeLists.txt b/support-files/CMakeLists.txt
index 5e9fd081a09..cc9eed6c7be 100644
--- a/support-files/CMakeLists.txt
+++ b/support-files/CMakeLists.txt
@@ -41,12 +41,14 @@ ELSE()
   SET(inst_location ${INSTALL_SUPPORTFILESDIR})
 ENDIF()
 
+IF (NOT WITHOUT_SERVER)
 FOREACH(inifile my-huge my-innodb-heavy-4G my-large my-medium my-small)
   CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/${inifile}.cnf.sh 
                  ${CMAKE_CURRENT_BINARY_DIR}/${inifile}.${ini_file_extension} @ONLY)
   INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/${inifile}.${ini_file_extension}
           DESTINATION ${inst_location} COMPONENT IniFiles)
 ENDFOREACH()
+ENDIF()
 
 IF(UNIX)
   SET(prefix ${CMAKE_INSTALL_PREFIX})

From 40f29ecbf1c8bf36be443c40382fe8cdc8a7f413 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Sat, 14 Jul 2018 00:51:23 +0200
Subject: [PATCH 045/164] MDEV-13397 MariaDB upgrade fail when using
 default_time_zone

don't try to set default time zone in --bootstrap,
this generally cannot be done, as timezone tables aren't loaded.
and bootstrap scripts don't need it anyway.
---
 mysql-test/t/bootstrap.test | 9 +++++++++
 sql/tztime.cc               | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/mysql-test/t/bootstrap.test b/mysql-test/t/bootstrap.test
index 5ab736cee15..3df3919220e 100644
--- a/mysql-test/t/bootstrap.test
+++ b/mysql-test/t/bootstrap.test
@@ -105,3 +105,12 @@ use test;
 EOF
 --exec $MYSQLD_BOOTSTRAP_CMD --ignore-db-dirs='some_dir' --ignore-db-dirs='some_dir' < $MYSQLTEST_VARDIR/tmp/bootstrap_9969.sql >> $MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1
 --remove_file $MYSQLTEST_VARDIR/tmp/bootstrap_9969.sql
+
+#
+# MDEV-13397 MariaDB upgrade fail when using default_time_zone
+#
+--write_file $MYSQLTEST_VARDIR/tmp/bootstrap_9969.sql
+use test;
+EOF
+--exec $MYSQLD_BOOTSTRAP_CMD --default-time-zone=Europe/Moscow < $MYSQLTEST_VARDIR/tmp/bootstrap_9969.sql >> $MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1
+--remove_file $MYSQLTEST_VARDIR/tmp/bootstrap_9969.sql
diff --git a/sql/tztime.cc b/sql/tztime.cc
index 75b732f4436..9933c1e6364 100644
--- a/sql/tztime.cc
+++ b/sql/tztime.cc
@@ -1663,7 +1663,7 @@ my_tz_init(THD *org_thd, const char *default_tzname, my_bool bootstrap)
   {
     /* If we are in bootstrap mode we should not load time zone tables */
     return_val= time_zone_tables_exist= 0;
-    goto end_with_setting_default_tz;
+    goto end_with_cleanup;
   }
 
   /*

From 5c744bb5350acbf2d3abce28d01fc829c4f2375c Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Sat, 14 Jul 2018 13:48:50 +0200
Subject: [PATCH 046/164] MDEV-14882 mysql_upgrade performs unnecessary
 conversions back and forth

avoid round-robin conversions, if the column is MODIFY-ed,
it should always be modified to its final definition, not to some
intermediate state.

also avoid other unconditional changes, like
  ALTER TABLE event DROP PRIMARY KEY;
  ALTER TABLE event ADD PRIMARY KEY(db, name);
---
 scripts/mysql_system_tables_fix.sql | 50 +++++++++++++++++------------
 1 file changed, 30 insertions(+), 20 deletions(-)

diff --git a/scripts/mysql_system_tables_fix.sql b/scripts/mysql_system_tables_fix.sql
index 1bb601ef5f9..194b1615c2b 100644
--- a/scripts/mysql_system_tables_fix.sql
+++ b/scripts/mysql_system_tables_fix.sql
@@ -1,5 +1,5 @@
 -- Copyright (C) 2003, 2013 Oracle and/or its affiliates.
--- Copyright (C) 2010, 2014 SkySQL Ab.
+-- Copyright (C) 2010, 2018 MariaDB Corporation
 --
 -- This program is free software; you can redistribute it and/or modify
 -- it under the terms of the GNU General Public License as published by
@@ -27,15 +27,24 @@
 set sql_mode='';
 set storage_engine=MyISAM;
 
-ALTER TABLE user add File_priv enum('N','Y') COLLATE utf8_general_ci NOT NULL;
+ALTER TABLE user add File_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL;
 
 # Detect whether or not we had the Grant_priv column
 SET @hadGrantPriv:=0;
 SELECT @hadGrantPriv:=1 FROM user WHERE Grant_priv LIKE '%';
 
-ALTER TABLE user add Grant_priv enum('N','Y') COLLATE utf8_general_ci NOT NULL,add References_priv enum('N','Y') COLLATE utf8_general_ci NOT NULL,add Index_priv enum('N','Y') COLLATE utf8_general_ci NOT NULL,add Alter_priv enum('N','Y') COLLATE utf8_general_ci NOT NULL;
-ALTER TABLE host add Grant_priv enum('N','Y') NOT NULL,add References_priv enum('N','Y') COLLATE utf8_general_ci NOT NULL,add Index_priv enum('N','Y') COLLATE utf8_general_ci NOT NULL,add Alter_priv enum('N','Y') COLLATE utf8_general_ci NOT NULL;
-ALTER TABLE db add Grant_priv enum('N','Y') COLLATE utf8_general_ci NOT NULL,add References_priv enum('N','Y') COLLATE utf8_general_ci NOT NULL,add Index_priv enum('N','Y') COLLATE utf8_general_ci NOT NULL,add Alter_priv enum('N','Y') COLLATE utf8_general_ci NOT NULL;
+ALTER TABLE user add Grant_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL,
+                 add References_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL,
+                 add Index_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL,
+                 add Alter_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL;
+ALTER TABLE host add Grant_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL,
+                 add References_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL,
+                 add Index_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL,
+                 add Alter_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL;
+ALTER TABLE db add Grant_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL,
+               add References_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL,
+               add Index_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL,
+               add Alter_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL;
 
 # Fix privileges for old tables
 UPDATE user SET Grant_priv=File_priv,References_priv=Create_priv,Index_priv=Create_priv,Alter_priv=Create_priv WHERE @hadGrantPriv = 0;
@@ -47,11 +56,11 @@ UPDATE host SET References_priv=Create_priv,Index_priv=Create_priv,Alter_priv=Cr
 # Adding columns needed by GRANT .. REQUIRE (openssl)
 
 ALTER TABLE user
-ADD ssl_type enum('','ANY','X509', 'SPECIFIED') COLLATE utf8_general_ci NOT NULL,
+ADD ssl_type enum('','ANY','X509', 'SPECIFIED') DEFAULT '' NOT NULL,
 ADD ssl_cipher BLOB NOT NULL,
 ADD x509_issuer BLOB NOT NULL,
 ADD x509_subject BLOB NOT NULL;
-ALTER TABLE user MODIFY ssl_type enum('','ANY','X509', 'SPECIFIED') NOT NULL;
+ALTER TABLE user MODIFY ssl_type enum('','ANY','X509', 'SPECIFIED') DEFAULT '' NOT NULL;
 
 #
 # tables_priv
@@ -62,9 +71,9 @@ ALTER TABLE tables_priv
 ALTER TABLE tables_priv
   MODIFY Host char(60) NOT NULL default '',
   MODIFY Db char(64) NOT NULL default '',
-  MODIFY User char(80) NOT NULL default '',
+  MODIFY User char(80) binary NOT NULL default '',
   MODIFY Table_name char(64) NOT NULL default '',
-  MODIFY Grantor char(141) NOT NULL default '',
+  MODIFY Grantor char(141) COLLATE utf8_bin NOT NULL default '',
   ENGINE=MyISAM,
   CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin;
 
@@ -90,7 +99,7 @@ ALTER TABLE columns_priv
 ALTER TABLE columns_priv
   MODIFY Host char(60) NOT NULL default '',
   MODIFY Db char(64) NOT NULL default '',
-  MODIFY User char(80) NOT NULL default '',
+  MODIFY User char(80) binary NOT NULL default '',
   MODIFY Table_name char(64) NOT NULL default '',
   MODIFY Column_name char(64) NOT NULL default '',
   ENGINE=MyISAM,
@@ -161,7 +170,7 @@ alter table func comment='User defined functions';
 # and reset all char columns to correct width
 ALTER TABLE user
   MODIFY Host char(60) NOT NULL default '',
-  MODIFY User char(80) NOT NULL default '',
+  MODIFY User char(80) binary NOT NULL default '',
   ENGINE=MyISAM, CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin;
 ALTER TABLE user
   MODIFY Password char(41) character set latin1 collate latin1_bin NOT NULL default '',
@@ -191,7 +200,7 @@ ALTER TABLE user
 ALTER TABLE db
   MODIFY Host char(60) NOT NULL default '',
   MODIFY Db char(64) NOT NULL default '',
-  MODIFY User char(80) NOT NULL default '',
+  MODIFY User char(80) binary NOT NULL default '',
   ENGINE=MyISAM, CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin;
 ALTER TABLE db
   MODIFY  Select_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL,
@@ -451,7 +460,7 @@ ALTER TABLE proc  MODIFY db
                   MODIFY definer
                          char(141) collate utf8_bin DEFAULT '' NOT NULL,
                   MODIFY comment
-                         char(64) collate utf8_bin DEFAULT '' NOT NULL;
+                         text collate utf8_bin NOT NULL;
 
 ALTER TABLE proc ADD character_set_client
                      char(32) collate utf8_bin DEFAULT NULL
@@ -515,19 +524,18 @@ ALTER TABLE proc MODIFY comment
 SET @hadEventPriv := 0;
 SELECT @hadEventPriv :=1 FROM user WHERE Event_priv LIKE '%';
 
-ALTER TABLE user add Event_priv enum('N','Y') character set utf8 DEFAULT 'N' NOT NULL AFTER Create_user_priv;
+ALTER TABLE user ADD Event_priv enum('N','Y') character set utf8 DEFAULT 'N' NOT NULL AFTER Create_user_priv;
 ALTER TABLE user MODIFY Event_priv enum('N','Y') character set utf8 DEFAULT 'N' NOT NULL AFTER Create_user_priv;
 
 UPDATE user SET Event_priv=Super_priv WHERE @hadEventPriv = 0;
 
-ALTER TABLE db add Event_priv enum('N','Y') character set utf8 DEFAULT 'N' NOT NULL;
+ALTER TABLE db ADD Event_priv enum('N','Y') character set utf8 DEFAULT 'N' NOT NULL;
 ALTER TABLE db MODIFY Event_priv enum('N','Y') character set utf8 DEFAULT 'N' NOT NULL;
 
 #
 # EVENT table
 #
-ALTER TABLE event DROP PRIMARY KEY;
-ALTER TABLE event ADD PRIMARY KEY(db, name);
+ALTER TABLE event DROP PRIMARY KEY, ADD PRIMARY KEY(db, name);
 # Add sql_mode column just in case.
 ALTER TABLE event ADD sql_mode set ('IGNORE_BAD_TABLE_OPTIONS') AFTER on_completion;
 # Update list of sql_mode values.
@@ -567,8 +575,8 @@ ALTER TABLE event MODIFY sql_mode
                             ) DEFAULT '' NOT NULL AFTER on_completion;
 ALTER TABLE event MODIFY name char(64) CHARACTER SET utf8 NOT NULL default '';
 
-ALTER TABLE event MODIFY COLUMN originator INT UNSIGNED NOT NULL;
 ALTER TABLE event ADD COLUMN originator INT UNSIGNED NOT NULL AFTER comment;
+ALTER TABLE event MODIFY COLUMN originator INT UNSIGNED NOT NULL;
 
 ALTER TABLE event MODIFY COLUMN status ENUM('ENABLED','DISABLED','SLAVESIDE_DISABLED') NOT NULL default 'ENABLED';
 
@@ -630,10 +638,12 @@ ALTER TABLE user MODIFY Create_tablespace_priv enum('N','Y') COLLATE utf8_genera
 
 UPDATE user SET Create_tablespace_priv = Super_priv WHERE @hadCreateTablespacePriv = 0;
 
-ALTER TABLE user ADD plugin char(64) DEFAULT '',  ADD authentication_string TEXT;
+ALTER TABLE user ADD plugin char(64) CHARACTER SET latin1 DEFAULT '' NOT NULL,
+                 ADD authentication_string TEXT NOT NULL;
+ALTER TABLE user MODIFY plugin char(64) CHARACTER SET latin1 DEFAULT '' NOT NULL,
+                 MODIFY authentication_string TEXT NOT NULL;
 ALTER TABLE user ADD password_expired ENUM('N', 'Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL;
 ALTER TABLE user ADD is_role enum('N', 'Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL;
-ALTER TABLE user MODIFY plugin char(64) CHARACTER SET latin1 DEFAULT '' NOT NULL, MODIFY authentication_string TEXT NOT NULL;
 -- Somewhere above, we ran ALTER TABLE user .... CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin.
 --  we want password_expired column to have collation utf8_general_ci.
 ALTER TABLE user MODIFY password_expired ENUM('N', 'Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL;

From d57ddaa1904d3c2f08460c539c72652f0c78c09a Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Mon, 16 Jul 2018 15:12:38 +0200
Subject: [PATCH 047/164] MDEV-15551 Server hangs or assertion
 `strcmp(share->unique_file_name,filename) || share->last_version' fails in
 test_if_reopen or unexpected ER_LOCK_DEADLOCK

only use HA_EXTRA_PREPARE_FOR_DROP when the table is going to be dropped
---
 mysql-test/suite/parts/r/truncate_locked.result |  7 +++++++
 mysql-test/suite/parts/t/truncate_locked.test   | 10 ++++++++++
 sql/sql_truncate.cc                             |  3 ++-
 3 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100644 mysql-test/suite/parts/r/truncate_locked.result
 create mode 100644 mysql-test/suite/parts/t/truncate_locked.test

diff --git a/mysql-test/suite/parts/r/truncate_locked.result b/mysql-test/suite/parts/r/truncate_locked.result
new file mode 100644
index 00000000000..54a3722938e
--- /dev/null
+++ b/mysql-test/suite/parts/r/truncate_locked.result
@@ -0,0 +1,7 @@
+create table t1 (i int) engine=myisam partition by hash(i) partitions 2 ;
+lock table t1 write;
+truncate table t1;
+desc t1;
+Field	Type	Null	Key	Default	Extra
+i	int(11)	YES		NULL	
+drop table t1;
diff --git a/mysql-test/suite/parts/t/truncate_locked.test b/mysql-test/suite/parts/t/truncate_locked.test
new file mode 100644
index 00000000000..4ff1016fb05
--- /dev/null
+++ b/mysql-test/suite/parts/t/truncate_locked.test
@@ -0,0 +1,10 @@
+#
+# MDEV-15551 Server hangs or assertion `strcmp(share->unique_file_name,filename) || share->last_version' fails in test_if_reopen or unexpected ER_LOCK_DEADLOCK
+#
+--source include/have_partition.inc
+create table t1 (i int) engine=myisam partition by hash(i) partitions 2 ;
+lock table t1 write;
+truncate table t1;
+desc t1;
+drop table t1;
+
diff --git a/sql/sql_truncate.cc b/sql/sql_truncate.cc
index 75a0678928b..f5161c09b2c 100644
--- a/sql/sql_truncate.cc
+++ b/sql/sql_truncate.cc
@@ -400,7 +400,8 @@ bool Sql_cmd_truncate_table::lock_table(THD *thd, TABLE_LIST *table_ref,
   {
     DEBUG_SYNC(thd, "upgrade_lock_for_truncate");
     /* To remove the table from the cache we need an exclusive lock. */
-    if (wait_while_table_is_used(thd, table, HA_EXTRA_PREPARE_FOR_DROP))
+    if (wait_while_table_is_used(thd, table,
+          *hton_can_recreate ? HA_EXTRA_PREPARE_FOR_DROP : HA_EXTRA_NOT_USED))
       DBUG_RETURN(TRUE);
     m_ticket_downgrade= table->mdl_ticket;
     /* Close if table is going to be recreated. */

From 323f269d4099b4a47530f0574af63292dfd89b8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Thu, 19 Jul 2018 15:13:31 +0300
Subject: [PATCH 048/164] MDEV-10564: Galera `wsrep_debug` patch logs MySQL
 user credentials

Restricted output for CREATE USER, GRANT, REVOKE and SET PASSWORD
so that it shows only above keywords but not rest of query i.e.
not user or password.
---
 sql/wsrep_mysqld.cc | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc
index b127d3a8f00..a88713652ed 100644
--- a/sql/wsrep_mysqld.cc
+++ b/sql/wsrep_mysqld.cc
@@ -1450,6 +1450,25 @@ static bool wsrep_can_run_in_toi(THD *thd, const char *db, const char *table,
   }
 }
 
+static const char* wsrep_get_query_or_msg(const THD* thd)
+{
+  switch(thd->lex->sql_command)
+  {
+    case SQLCOM_CREATE_USER:
+      return "CREATE USER";
+    case SQLCOM_GRANT:
+      return "GRANT";
+    case SQLCOM_REVOKE:
+      return "REVOKE";
+    case SQLCOM_SET_OPTION:
+      if (thd->lex->definer)
+	return "SET PASSWORD";
+      /* fallthrough */
+    default:
+      return thd->query();
+   }
+}
+
 /*
   returns: 
    0: statement was replicated as TOI
@@ -1472,7 +1491,8 @@ static int wsrep_TOI_begin(THD *thd, char *db_, char *table_,
   }
 
   WSREP_DEBUG("TO BEGIN: %lld, %d : %s", (long long)wsrep_thd_trx_seqno(thd),
-              thd->wsrep_exec_mode, thd->query() );
+              thd->wsrep_exec_mode, wsrep_get_query_or_msg(thd));
+
   switch (thd->lex->sql_command)
   {
   case SQLCOM_CREATE_VIEW:
@@ -1546,8 +1566,8 @@ static void wsrep_TOI_end(THD *thd) {
   wsrep_status_t ret;
   wsrep_to_isolation--;
 
-  WSREP_DEBUG("TO END: %lld, %d : %s", (long long)wsrep_thd_trx_seqno(thd),
-              thd->wsrep_exec_mode, (thd->query()) ? thd->query() : "void");
+  WSREP_DEBUG("TO END: %lld, %d: %s", (long long)wsrep_thd_trx_seqno(thd),
+              thd->wsrep_exec_mode, wsrep_get_query_or_msg(thd));
 
   wsrep_set_SE_checkpoint(thd->wsrep_trx_meta.gtid.uuid,
                           thd->wsrep_trx_meta.gtid.seqno);

From 9cea4ccf12cb6e8746b9b440d9c62408a9ef04af Mon Sep 17 00:00:00 2001
From: Igor Babaev <igor@askmonty.org>
Date: Thu, 19 Jul 2018 15:31:30 -0700
Subject: [PATCH 049/164] MDEV-16726 Assertion `tab->type == JT_REF ||
 tab->type == JT_EQ_REF' failed

Due to a legacy bug in the code of make_join_statistics() detecting
so-called constant tables could miss some of them in rare queries
that used RIGHT JOIN. As a result these queries had execution plans
different from the execution plans of the equivalent queries with
LEFT JOIN.
Besides starting from 10.2 this could trigger an assertion failure.
---
 mysql-test/r/join_outer.result              | 50 +++++++++++++++++++++
 mysql-test/r/join_outer_jcl6.result         | 50 +++++++++++++++++++++
 mysql-test/r/subselect_mat_cost_bugs.result |  2 +-
 mysql-test/t/join_outer.test                | 48 ++++++++++++++++++++
 sql/sql_select.cc                           |  4 +-
 5 files changed, 151 insertions(+), 3 deletions(-)

diff --git a/mysql-test/r/join_outer.result b/mysql-test/r/join_outer.result
index 67b22ca86b2..d55f11cc2f2 100644
--- a/mysql-test/r/join_outer.result
+++ b/mysql-test/r/join_outer.result
@@ -2368,5 +2368,55 @@ id	sid	id
 1	NULL	NULL
 2	NULL	NULL
 drop table t1, t2;
+#
+# MDEV-16726: SELECT with STRAGHT JOIN containing NESTED RIGHT JOIN
+#             converted to INNER JOIN with first constant inner table
+#
+CREATE TABLE t1 (
+pk int PRIMARY KEY, i1 int, v1 varchar(1), v2 varchar(1), KEY v1 (v1,i1)
+) engine=MyISAM;
+INSERT INTO t1 VALUES
+(8,3,'c','c'),(9,4,'z','z'),(10,3,'i','i'),(11,186,'x','x'),
+(14,226,'m','m'),(15,133,'p','p');
+CREATE TABLE t2 (
+pk int PRIMARY KEY, i1 int, v1 varchar(1), v2 varchar(1)
+) engine=MyISAM;
+INSERT INTO t2 VALUES (10,6,'p','p');
+EXPLAIN EXTENDED
+SELECT STRAIGHT_JOIN t2.v2
+FROM
+(t1 as tb1 LEFT JOIN t1 AS tb2 ON tb2.v1 = tb1.v2)
+RIGHT JOIN
+(t2,t1)
+ON t1.pk = t2.pk AND t2.v2 = tb1.v1
+WHERE tb1.pk = 40
+ORDER BY tb1.i1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE noticed after reading const tables
+Warnings:
+Note	1003	select straight_join 'p' AS `v2` from `test`.`t1` join `test`.`t1` `tb1` left join `test`.`t1` `tb2` on(multiple equal(`test`.`tb2`.`v1`, NULL)) where 0 order by NULL
+EXPLAIN EXTENDED
+SELECT STRAIGHT_JOIN t2.v2
+FROM
+(t2,t1)
+LEFT JOIN
+(t1 as tb1 LEFT JOIN t1 AS tb2 ON tb2.v1 = tb1.v2)
+ON t1.pk = t2.pk AND t2.v2 = tb1.v1
+WHERE tb1.pk = 40
+ORDER BY tb1.i1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE noticed after reading const tables
+Warnings:
+Note	1003	select straight_join 'p' AS `v2` from `test`.`t1` join `test`.`t1` `tb1` left join `test`.`t1` `tb2` on(multiple equal(`test`.`tb2`.`v1`, NULL)) where 0 order by NULL
+SELECT STRAIGHT_JOIN DISTINCT t2.v2
+FROM
+(t1 as tb1 LEFT JOIN t1 AS tb2 ON tb2.v1 = tb1.v2)
+RIGHT JOIN
+(t2,t1)
+ON t1.pk = t2.pk AND t2.v2 = tb1.v1
+WHERE tb1.pk = 40
+ORDER BY tb1.i1;
+v2
+DROP TABLE t1,t2;
 # end of 5.5 tests
 SET optimizer_switch=@save_optimizer_switch;
diff --git a/mysql-test/r/join_outer_jcl6.result b/mysql-test/r/join_outer_jcl6.result
index c019da6197b..8a9b395edff 100644
--- a/mysql-test/r/join_outer_jcl6.result
+++ b/mysql-test/r/join_outer_jcl6.result
@@ -2379,6 +2379,56 @@ id	sid	id
 1	NULL	NULL
 2	NULL	NULL
 drop table t1, t2;
+#
+# MDEV-16726: SELECT with STRAGHT JOIN containing NESTED RIGHT JOIN
+#             converted to INNER JOIN with first constant inner table
+#
+CREATE TABLE t1 (
+pk int PRIMARY KEY, i1 int, v1 varchar(1), v2 varchar(1), KEY v1 (v1,i1)
+) engine=MyISAM;
+INSERT INTO t1 VALUES
+(8,3,'c','c'),(9,4,'z','z'),(10,3,'i','i'),(11,186,'x','x'),
+(14,226,'m','m'),(15,133,'p','p');
+CREATE TABLE t2 (
+pk int PRIMARY KEY, i1 int, v1 varchar(1), v2 varchar(1)
+) engine=MyISAM;
+INSERT INTO t2 VALUES (10,6,'p','p');
+EXPLAIN EXTENDED
+SELECT STRAIGHT_JOIN t2.v2
+FROM
+(t1 as tb1 LEFT JOIN t1 AS tb2 ON tb2.v1 = tb1.v2)
+RIGHT JOIN
+(t2,t1)
+ON t1.pk = t2.pk AND t2.v2 = tb1.v1
+WHERE tb1.pk = 40
+ORDER BY tb1.i1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE noticed after reading const tables
+Warnings:
+Note	1003	select straight_join 'p' AS `v2` from `test`.`t1` join `test`.`t1` `tb1` left join `test`.`t1` `tb2` on(multiple equal(`test`.`tb2`.`v1`, NULL)) where 0 order by NULL
+EXPLAIN EXTENDED
+SELECT STRAIGHT_JOIN t2.v2
+FROM
+(t2,t1)
+LEFT JOIN
+(t1 as tb1 LEFT JOIN t1 AS tb2 ON tb2.v1 = tb1.v2)
+ON t1.pk = t2.pk AND t2.v2 = tb1.v1
+WHERE tb1.pk = 40
+ORDER BY tb1.i1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE noticed after reading const tables
+Warnings:
+Note	1003	select straight_join 'p' AS `v2` from `test`.`t1` join `test`.`t1` `tb1` left join `test`.`t1` `tb2` on(multiple equal(`test`.`tb2`.`v1`, NULL)) where 0 order by NULL
+SELECT STRAIGHT_JOIN DISTINCT t2.v2
+FROM
+(t1 as tb1 LEFT JOIN t1 AS tb2 ON tb2.v1 = tb1.v2)
+RIGHT JOIN
+(t2,t1)
+ON t1.pk = t2.pk AND t2.v2 = tb1.v1
+WHERE tb1.pk = 40
+ORDER BY tb1.i1;
+v2
+DROP TABLE t1,t2;
 # end of 5.5 tests
 SET optimizer_switch=@save_optimizer_switch;
 set join_cache_level=default;
diff --git a/mysql-test/r/subselect_mat_cost_bugs.result b/mysql-test/r/subselect_mat_cost_bugs.result
index d33f1488e4d..03f4b3e1903 100644
--- a/mysql-test/r/subselect_mat_cost_bugs.result
+++ b/mysql-test/r/subselect_mat_cost_bugs.result
@@ -442,7 +442,7 @@ SELECT i2 FROM t2 RIGHT JOIN t3 ON (c3 = c2) WHERE pk3 = i1
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	system	NULL	NULL	NULL	NULL	1	
 2	DEPENDENT SUBQUERY	t3	const	PRIMARY	PRIMARY	4	const	1	
-2	DEPENDENT SUBQUERY	t2	index	NULL	i2	11	NULL	2	Using where; Using index
+2	DEPENDENT SUBQUERY	t2	index	i2	i2	11	NULL	2	Using where; Using index
 DROP TABLE t1,t2,t3;
 #
 # MDEV-7599: in-to-exists chosen after min/max optimization
diff --git a/mysql-test/t/join_outer.test b/mysql-test/t/join_outer.test
index 2769aea9969..305421c10d5 100644
--- a/mysql-test/t/join_outer.test
+++ b/mysql-test/t/join_outer.test
@@ -1911,6 +1911,54 @@ select * from t1 t
  on t.id=r.id ;
 drop table t1, t2;
 
+--echo #
+--echo # MDEV-16726: SELECT with STRAGHT JOIN containing NESTED RIGHT JOIN
+--echo #             converted to INNER JOIN with first constant inner table
+--echo #
+
+CREATE TABLE t1 (
+  pk int PRIMARY KEY, i1 int, v1 varchar(1), v2 varchar(1), KEY v1 (v1,i1)
+) engine=MyISAM;
+INSERT INTO t1 VALUES
+  (8,3,'c','c'),(9,4,'z','z'),(10,3,'i','i'),(11,186,'x','x'),
+  (14,226,'m','m'),(15,133,'p','p');
+
+CREATE TABLE t2 (
+  pk int PRIMARY KEY, i1 int, v1 varchar(1), v2 varchar(1)
+) engine=MyISAM;
+INSERT INTO t2 VALUES (10,6,'p','p');
+
+EXPLAIN EXTENDED
+SELECT STRAIGHT_JOIN t2.v2
+FROM
+  (t1 as tb1 LEFT JOIN t1 AS tb2 ON tb2.v1 = tb1.v2)
+  RIGHT JOIN
+  (t2,t1)
+  ON t1.pk = t2.pk AND t2.v2 = tb1.v1
+WHERE tb1.pk = 40
+ORDER BY tb1.i1;
+
+EXPLAIN EXTENDED
+SELECT STRAIGHT_JOIN t2.v2
+FROM
+    (t2,t1)
+    LEFT JOIN
+    (t1 as tb1 LEFT JOIN t1 AS tb2 ON tb2.v1 = tb1.v2)
+   ON t1.pk = t2.pk AND t2.v2 = tb1.v1
+WHERE tb1.pk = 40
+ORDER BY tb1.i1;
+
+SELECT STRAIGHT_JOIN DISTINCT t2.v2
+FROM
+  (t1 as tb1 LEFT JOIN t1 AS tb2 ON tb2.v1 = tb1.v2)
+  RIGHT JOIN
+  (t2,t1)
+  ON t1.pk = t2.pk AND t2.v2 = tb1.v1
+WHERE tb1.pk = 40
+ORDER BY tb1.i1;
+
+DROP TABLE t1,t2;
+
 --echo # end of 5.5 tests
 
 SET optimizer_switch=@save_optimizer_switch;
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 700b7b37928..6a64a0e9952 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -3456,8 +3456,8 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
   int ref_changed;
   do
   {
-  more_const_tables_found:
     ref_changed = 0;
+  more_const_tables_found:
     found_ref=0;
 
     /*
@@ -3622,7 +3622,7 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
 	}
       }
     }
-  } while (join->const_table_map & found_ref && ref_changed);
+  } while (ref_changed);
  
   join->sort_by_table= get_sort_by_table(join->order, join->group_list,
                                          join->select_lex->leaf_tables,

From a0d33dc6ef6f380fb5a2d166e75ed6fed2dfce9d Mon Sep 17 00:00:00 2001
From: Oleksandr Byelkin <sanja@mariadb.com>
Date: Fri, 20 Jul 2018 13:35:58 +0200
Subject: [PATCH 050/164] MDEV-16689: core-file should become a real server
 variable

Mysql Bug#56124 bug analog fix using Sys_var_bit.
---
 client/mysqlbinlog.cc                              |  2 +-
 .../sys_vars/r/sysvars_server_embedded.result      | 14 ++++++++++++++
 .../sys_vars/r/sysvars_server_notembedded.result   | 14 ++++++++++++++
 sql/mysqld.cc                                      |  3 ++-
 sql/mysqld.h                                       |  3 ++-
 sql/sys_vars.cc                                    |  6 ++++++
 6 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc
index 9753125dd67..bc13aa6c2cc 100644
--- a/client/mysqlbinlog.cc
+++ b/client/mysqlbinlog.cc
@@ -71,7 +71,7 @@ ulong bytes_sent = 0L, bytes_received = 0L;
 ulong mysqld_net_retry_count = 10L;
 ulong open_files_limit;
 ulong opt_binlog_rows_event_max_size;
-uint test_flags = 0; 
+ulonglong test_flags = 0;
 static uint opt_protocol= 0;
 static FILE *result_file;
 static char *result_file_name= 0;
diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result
index 7f046a188cc..2b3ecc36772 100644
--- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result
+++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result
@@ -471,6 +471,20 @@ NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
+VARIABLE_NAME	CORE_FILE
+SESSION_VALUE	NULL
+GLOBAL_VALUE	ON
+GLOBAL_VALUE_ORIGIN	COMPILE-TIME
+DEFAULT_VALUE	NULL
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	BOOLEAN
+VARIABLE_COMMENT	write a core-file on crashes
+NUMERIC_MIN_VALUE	NULL
+NUMERIC_MAX_VALUE	NULL
+NUMERIC_BLOCK_SIZE	NULL
+ENUM_VALUE_LIST	OFF,ON
+READ_ONLY	YES
+COMMAND_LINE_ARGUMENT	NULL
 VARIABLE_NAME	DATADIR
 SESSION_VALUE	NULL
 GLOBAL_VALUE	PATH
diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result
index 72550ffb3af..8df07cdefbd 100644
--- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result
+++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result
@@ -471,6 +471,20 @@ NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
+VARIABLE_NAME	CORE_FILE
+SESSION_VALUE	NULL
+GLOBAL_VALUE	ON
+GLOBAL_VALUE_ORIGIN	COMPILE-TIME
+DEFAULT_VALUE	NULL
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	BOOLEAN
+VARIABLE_COMMENT	write a core-file on crashes
+NUMERIC_MIN_VALUE	NULL
+NUMERIC_MAX_VALUE	NULL
+NUMERIC_BLOCK_SIZE	NULL
+ENUM_VALUE_LIST	OFF,ON
+READ_ONLY	YES
+COMMAND_LINE_ARGUMENT	NULL
 VARIABLE_NAME	DATADIR
 SESSION_VALUE	NULL
 GLOBAL_VALUE	PATH
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index 0f5147c8cf1..1e6e144ccff 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -482,7 +482,7 @@ my_bool opt_master_verify_checksum= 0;
 my_bool opt_slave_sql_verify_checksum= 1;
 const char *binlog_format_names[]= {"MIXED", "STATEMENT", "ROW", NullS};
 volatile sig_atomic_t calling_initgroups= 0; /**< Used in SIGSEGV handler. */
-uint mysqld_port, test_flags, select_errors, dropping_tables, ha_open_options;
+uint mysqld_port, select_errors, dropping_tables, ha_open_options;
 uint mysqld_extra_port;
 uint mysqld_port_timeout;
 ulong delay_key_write_options;
@@ -509,6 +509,7 @@ ulonglong max_binlog_cache_size=0;
 ulong slave_max_allowed_packet= 0;
 ulonglong binlog_stmt_cache_size=0;
 ulonglong  max_binlog_stmt_cache_size=0;
+ulonglong test_flags;
 ulonglong query_cache_size=0;
 ulong query_cache_limit=0;
 ulong executed_events=0;
diff --git a/sql/mysqld.h b/sql/mysqld.h
index 97ecf889787..0a3266abfb5 100644
--- a/sql/mysqld.h
+++ b/sql/mysqld.h
@@ -159,7 +159,8 @@ extern ulong opt_tc_log_size, tc_log_max_pages_used, tc_log_page_size;
 extern ulong tc_log_page_waits;
 extern my_bool relay_log_purge, opt_innodb_safe_binlog, opt_innodb;
 extern my_bool relay_log_recovery;
-extern uint test_flags,select_errors,ha_open_options;
+extern uint select_errors,ha_open_options;
+extern ulonglong test_flags;
 extern uint protocol_version, mysqld_port, dropping_tables;
 extern ulong delay_key_write_options;
 extern char *opt_logname, *opt_slow_logname, *opt_bin_logname, 
diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc
index 5470acc7892..f941794f89e 100644
--- a/sql/sys_vars.cc
+++ b/sql/sys_vars.cc
@@ -440,6 +440,12 @@ bool check_has_super(sys_var *self, THD *thd, set_var *var)
 #endif
   return false;
 }
+
+static Sys_var_bit Sys_core_file("core_file", "write a core-file on crashes",
+          READ_ONLY GLOBAL_VAR(test_flags), NO_CMD_LINE,
+          TEST_CORE_ON_SIGNAL, DEFAULT(FALSE), NO_MUTEX_GUARD, NOT_IN_BINLOG,
+          0,0,0);
+
 static bool binlog_format_check(sys_var *self, THD *thd, set_var *var)
 {
   if (check_has_super(self, thd, var))

From e0139c2b92ba1f8819a3ed75d020d9665545e2bf Mon Sep 17 00:00:00 2001
From: Oleksandr Byelkin <sanja@mariadb.com>
Date: Tue, 24 Jul 2018 18:16:41 +0200
Subject: [PATCH 051/164] fix plugins.processlist

make it not to fail when `show engine innodb status` output
contains a double quote
---
 mysql-test/suite/plugins/t/processlist.test | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/mysql-test/suite/plugins/t/processlist.test b/mysql-test/suite/plugins/t/processlist.test
index 5aacef317b9..e8f03aacb10 100644
--- a/mysql-test/suite/plugins/t/processlist.test
+++ b/mysql-test/suite/plugins/t/processlist.test
@@ -8,7 +8,8 @@ start transaction;
 insert t1 values (1);
 let id=`select connection_id()`;
 connect con2,localhost,root;
-let s=query_get_value(show engine innodb status,Status,1);
+replace_regex /\"/-/; #"
+let s=`show engine innodb status`;
 disable_query_log;
 eval select regexp_replace("$s", '(?s)^.*MySQL thread id $id,.*root([^\n]*)\n.*', '\\\\1') as `state from show engine innodb status`;
 eval select state as `state from show processlist` from information_schema.processlist where id = $id;

From 9fbe360e9f7d41d169eb5abed22dc15f26f6c749 Mon Sep 17 00:00:00 2001
From: Oleksandr Byelkin <sanja@mariadb.com>
Date: Tue, 24 Jul 2018 18:24:21 +0200
Subject: [PATCH 052/164] make plugins.processlist more robust

---
 mysql-test/suite/plugins/r/processlist.result | 4 +---
 mysql-test/suite/plugins/t/processlist.test   | 5 +++--
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/mysql-test/suite/plugins/r/processlist.result b/mysql-test/suite/plugins/r/processlist.result
index d08ea9d3523..9200c865533 100644
--- a/mysql-test/suite/plugins/r/processlist.result
+++ b/mysql-test/suite/plugins/r/processlist.result
@@ -1,8 +1,6 @@
 create table t1 (a int) engine=innodb;
 start transaction;
 insert t1 values (1);
-state from show engine innodb status
+state from show engine innodb status, must be empty
  
-state from show processlist
-
 drop table t1;
diff --git a/mysql-test/suite/plugins/t/processlist.test b/mysql-test/suite/plugins/t/processlist.test
index e8f03aacb10..39b715b867b 100644
--- a/mysql-test/suite/plugins/t/processlist.test
+++ b/mysql-test/suite/plugins/t/processlist.test
@@ -8,11 +8,12 @@ start transaction;
 insert t1 values (1);
 let id=`select connection_id()`;
 connect con2,localhost,root;
+let $wait_condition=select state='' from information_schema.processlist where id = $id;
+--source include/wait_condition.inc
 replace_regex /\"/-/; #"
 let s=`show engine innodb status`;
 disable_query_log;
-eval select regexp_replace("$s", '(?s)^.*MySQL thread id $id,.*root([^\n]*)\n.*', '\\\\1') as `state from show engine innodb status`;
-eval select state as `state from show processlist` from information_schema.processlist where id = $id;
+eval select regexp_replace("$s", '(?s)^.*MySQL thread id $id,.*root([^\n]*)\n.*', '\\\\1') as `state from show engine innodb status, must be empty`;
 enable_query_log;
 disconnect con2;
 connection default;

From 1bda5e3a8f21d14b4af416259f293a92c49daebb Mon Sep 17 00:00:00 2001
From: Elena Stepanova <elenst@montyprogram.com>
Date: Tue, 24 Jul 2018 20:09:42 +0300
Subject: [PATCH 053/164] List of unstable tests for 10.0.36 release

---
 mysql-test/unstable-tests | 174 +++++++++++++++++++++-----------------
 1 file changed, 98 insertions(+), 76 deletions(-)

diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests
index d50a9acc92e..5fa6f6653c3 100644
--- a/mysql-test/unstable-tests
+++ b/mysql-test/unstable-tests
@@ -23,59 +23,79 @@
 #
 ##############################################################################
 
-# Based on 10.0 48636f09720af45454e5db1183b62a1991db8faf
+# Based on 10.0 5e67567b158db5d294b11b310ff35a454f58d034
 
+main.alter_table                   : Modified in 10.0.36
+main.assign_key_cache              : Added in 10.0.36
+main.assign_key_cache_debug        : Added in 10.0.36
+main.auto_increment                : Modified in 10.0.36
+main.bootstrap                     : Modified in 10.0.36
+main.connect_debug                 : Added in 10.0.36
 main.count_distinct2               : MDEV-11768 - timeout
 main.create_delayed                : MDEV-10605 - failed with timeout
-main.ctype_ucs                     : Modified in 10.0.35
-main.ctype_utf8mb4                 : Modified in 10.0.35
+main.create_or_replace             : Modified in 10.0.36
+ctype_binary                       : Modified in 10.0.36
+ctype_eucjpms                      : Modified in 10.0.36
+ctype_euckr                        : Modified in 10.0.36
+ctype_gbk                          : Modified in 10.0.36
+ctype_latin1                       : Modified in 10.0.36
+ctype_ucs                          : Modified in 10.0.36
+ctype_ujis                         : Modified in 10.0.36
+ctype_utf16le                      : Modified in 10.0.36
+ctype_utf16                        : Modified in 10.0.36
+ctype_utf32                        : Modified in 10.0.36
+ctype_utf8mb4                      : Modified in 10.0.36
+ctype_utf8                         : Modified in 10.0.36
 main.debug_sync                    : MDEV-10607 - internal error
+main.derived                       : Modified in 10.0.36
 main.derived_opt                   : MDEV-11768 - timeout
-main.dyncol                        : Modified in 10.0.35
 main.events_slowlog                : MDEV-12821 - wrong result
-main.func_concat                   : Modified in 10.0.35
-main.func_misc                     : Modified in 10.0.35
-main.func_str                      : Modified in 10.0.35
-main.func_time                     : Modified in 10.0.35
-main.having                        : Modified in 10.0.35
 main.gis                           : MDEV-13411 - wrong result on P8
+main.grant                         : Modified in 10.0.36
+main.grant2                        : Modified in 10.0.36
+main.grant_not_windows             : Added in 10.0.36
+main.having                        : Modified in 10.0.36
 main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown
 main.index_intersect_innodb        : MDEV-10643 - failed with timeout
 main.index_merge_innodb            : MDEV-7142 - wrong result
 main.innodb_mysql_lock             : MDEV-7861 - sporadic lock detection failure
-main.join_outer                    : Modified in 10.0.35
+main.insert_select                 : Modified in 10.0.36
+main.join                          : Modified in 10.0.36
+main.join_cache                    : Modified in 10.0.36
+main.join_outer                    : Modified in 10.0.36
 main.kill_processlist-6619         : MDEV-10793 - wrong result
+main.limit                         : Modified in 10.0.36
 main.log_tables-big                : MDEV-13408 - wrong result
 main.mdev-504                      : MDEV-10607 - sporadic "can't connect"
 main.mdev375                       : MDEV-10607 - sporadic "can't connect"
 main.merge                         : MDEV-10607 - sporadic "can't connect"
+main.myisam                        : Modified in 10.0.36
+main.mysql                         : Modified in 10.0.36
+main.mysql_cp932                   : Modified in 10.0.36
+main.mysqldump                     : Modified in 10.0.36
 main.mysqlhotcopy_myisam           : MDEV-10995 - test hangs on debug build
+main.mysqlslap                     : Modified in 10.0.36
 main.mysqltest                     : MDEV-9269 - fails on Alpha
 main.mysql_client_test_nonblock    : MDEV-15096 - exec failed
-main.parser                        : Modified in 10.0.35
-main.partition                     : Modified in 10.0.35
 main.ps                            : MDEV-11017 - sporadic wrong Prepared_stmt_count
-main.ps_qc_innodb                  : Added in 10.0.35
+main.rename                        : Modified in 10.0.36
 main.query_cache_debug             : MDEV-15281 - resize or similar command in progress
-main.read_only_innodb              : Modified in 10.0.35
+main.selectivity                   : Modified in 10.0.36
 main.show_explain                  : MDEV-10674 - wrong result
-main.sp-destruct                   : Modified in 10.0.35
+main.sp-innodb                     : Modified in 10.0.36
 main.sp_notembedded                : MDEV-10607 - internal error
 main.sp-security                   : MDEV-10607 - sporadic "can't connect"
+main.statistics                    : Modified in 10.0.36
+main.statistics_close              : Added in 10.0.36
+main.stat_tables                   : Modified in 10.0.36
 main.stat_tables_par_innodb        : MDEV-14155 - wrong rounding
-main.statistics                    : Modified in 10.0.35
 main.subselect_innodb              : MDEV-10614 - sporadic wrong results
-main.subselect_mat                 : Modified in 10.0.35
-main.subselect4                    : Modified in 10.0.35
-main.symlink-aria-11902            : MDEV-15098 - error 40 from storage engine
-main.symlink-myisam-11902          : MDEV-15098 - error 40 from storage engine
+main.subselect_sj                  : Modified in 10.0.36
+main.subselect_sj2_mat             : Modified in 10.0.36
+main.subselect4                    : Modified in 10.0.36
 main.tc_heuristic_recover          : MDEV-15200 - wrong error on mysqld_stub_cmd
 main.type_datetime                 : MDEV-14322 - wrong result
-main.type_time_6065                : Modified in 10.0.35
-main.update_innodb                 : Modified in 10.0.35
-main.view                          : Modified in 10.0.35
 main.xa                            : MDEV-11769 - lock wait timeout
-main.xml                           : Modified in 10.0.35
 
 #----------------------------------------------------------------
 
@@ -84,27 +104,16 @@ archive.archive_symlink      : MDEV-12170 - unexpected error on rmdir
 archive.discover             : MDEV-10510 - table is marked as crashed
 archive.mysqlhotcopy_archive : MDEV-10995 - test hangs on debug build, MDEV-14726 - table is marked as crashed
 
+archive-test_sql_discovery.discover : MDEV-16817 - Table marked as crashed
+
 #----------------------------------------------------------------
 
 binlog.binlog_commit_wait          : MDEV-10150 - Error: too much time elapsed
+binlog.binlog_tmp_table_row        : Added in 10.0.36
 binlog.binlog_xa_recover           : MDEV-8517 - Extra checkpoint
 
 #----------------------------------------------------------------
 
-connect.jdbc             : Included file modified in 10.0.35
-connect.jdbc_new         : Included file modified in 10.0.35
-connect.jdbc_oracle      : Included file modified in 10.0.35
-connect.jdbc_postgresql  : Modified in 10.0.35
-connect.json_java_2      : Included file modified in 10.0.35
-connect.json_java_3      : Included file modified in 10.0.35
-connect.json_mongo_c     : Included file modified in 10.0.35
-connect.json_udf         : Modified in 10.0.35
-connect.mongo_c          : Included file modified in 10.0.35
-connect.mongo_java_2     : Included file modified in 10.0.35
-connect.mongo_java_3     : Included file modified in 10.0.35
-connect.mongo_test       : Modified in 10.0.35
-connect.tbl_thread       : Modified in 10.0.35
-connect.vcol             : Added in 10.0.35
 connect.zip              : MDEV-13884 - wrong result
 
 #----------------------------------------------------------------
@@ -113,14 +122,16 @@ engines/rr_trx.* : MDEV-10998 - tests not maintained
 
 #----------------------------------------------------------------
 
-federated.federatedx             : MDEV-10617 - Wrong checksum, timeouts
+federated.assisted_discovery     : Include file modified in 10.0.36
+federated.federatedx             : MDEV-10617 - Wrong checksum, timeouts; include file modified in 10.0.36
 federated.federated_bug_35333    : MDEV-13410 - Wrong result
 federated.federated_innodb       : MDEV-10617, MDEV-10417 - Wrong checksum, timeouts, fails on Mips
-federated.federated_partition    : MDEV-10417 - Fails on Mips
-federated.federated_transactions : MDEV-10617, MDEV-10417 - Wrong checksum, timeouts, fails on Mips
+federated.federated_partition    : MDEV-10417 - Fails on Mips; include file modified in 10.0.36
+federated.federated_transactions : MDEV-10617, MDEV-10417 - Wrong checksum, timeouts, fails on Mips; include file modified in 10.0.36
 
 #----------------------------------------------------------------
 
+funcs_1.is_engines_federated : Include file modified in 10.0.36
 funcs_1.memory_views       : MDEV-11773 - timeout
 funcs_1.processlist_val_ps : MDEV-12175 - Wrong result
 funcs_1.processlist_val_no_prot : MDEV-11223 - Wrong result
@@ -129,34 +140,40 @@ funcs_2/charset.*          : MDEV-10999 - test not maintained
 
 #----------------------------------------------------------------
 
-innodb.binlog_consistent                     : MDEV-10618 - Server fails to start
-innodb.group_commit_crash                    : MDEV-11770 - checksum mismatch
-innodb.group_commit_crash_no_optimize_thread : MDEV-11770 - checksum mismatch
-innodb.innodb-alter-nullable                 : Modified in 10.0.35
-innodb.innodb-alter-table                    : MDEV-10619 - Testcase timeout
-innodb.innodb_bug11754376                    : Modified in 10.0.35
-innodb.innodb_bug27216817                    : Added in 10.0.35
-innodb.innodb_bug30423                       : MDEV-7311 - Wrong number of rows in the plan
-innodb.innodb_bug48024                       : MDEV-14352 - Assertion failure
-innodb.innodb_bug56947                       : Modified in 10.0.35
-innodb.innodb_corrupt_bit                    : Modified in 10.0.35
-innodb.innodb_monitor                        : MDEV-10939 - Testcase timeout
-innodb.innodb-replace-debug                  : Modified in 10.0.35
-innodb.log_file_size                         : MDEV-15668 - Not found pattern; modified in 10.0.35
-innodb.mvcc                                  : Added in 10.0.35
-innodb.read_only_recover_committed           : Added in 10.0.35
-innodb.recovery_shutdown                     : MDEV-15671 - Warning: database page corruption; modified in 10.0.35
-innodb.xa_recovery                           : MDEV-15279 - mysqld got exception
-
-innodb_fts.innobase_drop_fts_index_table     : Modified in 10.0.35
-innodb_fts.innodb-fts-fic                    : MDEV-14154 - Assertion failure
-innodb_fts.innodb_fts_misc_debug             : MDEV-14156 - Unexpected warning; modified in 10.0.35
-innodb_fts.innodb_fts_result_cache_limit     : Modified in 10.0.35
+handler.ps                 : Added in 10.0.36
 
 #----------------------------------------------------------------
 
-maria.dynamic                                : Added in 10.0.35
-maria.maria                                  : MDEV-14430 - Wrong result
+heap.heap_auto_increment   : Modified in 10.0.36
+
+#----------------------------------------------------------------
+
+innodb.alter_partitioned_xa                  : Added in 10.0.36
+innodb.binlog_consistent                     : MDEV-10618 - Server fails to start
+innodb.group_commit_crash                    : MDEV-11770 - checksum mismatch
+innodb.group_commit_crash_no_optimize_thread : MDEV-11770 - checksum mismatch
+innodb.innodb-alter                          : Modified in 10.0.36
+innodb.innodb-alter-table                    : MDEV-10619 - Testcase timeout
+innodb.innodb_bug30423                       : MDEV-7311 - Wrong number of rows in the plan
+innodb.innodb_bug48024                       : MDEV-14352 - Assertion failure
+innodb.innodb_bug54044                       : Modified in 10.0.36
+innodb.innodb-mdev7046                       : Modified in 10.0.36
+innodb.innodb_monitor                        : MDEV-10939 - Testcase timeout
+innodb.log_file_size                         : MDEV-15668 - Not found pattern
+innodb.recovery_shutdown                     : MDEV-15671 - Warning: database page corruption
+innodb.rename_table                          : Added in 10.0.36
+innodb.table_definition_cache_debug          : MDEV-14206 - Unexpected warning
+innodb.xa_recovery                           : MDEV-15279 - mysqld got exception
+
+innodb_fts.basic                             : Added in 10.0.36
+innodb_fts.innodb-fts-fic                    : MDEV-14154 - Assertion failure
+innodb_fts.innodb_fts_misc_debug             : MDEV-14156 - Unexpected warning
+
+#----------------------------------------------------------------
+
+maria.alter                                  : Modified in 10.0.36
+maria.lock                                   : Modified in 10.0.36
+maria.maria                                  : MDEV-14430 - Wrong result; modified in 10.0.36
 
 #----------------------------------------------------------------
 
@@ -178,12 +195,11 @@ multi_source.status_vars : MDEV-4632 - failed while waiting for Slave_received_h
 
 #----------------------------------------------------------------
 
-parts.partition_alter_innodb         : Include file modified in 10.0.35
-parts.partition_alter_maria          : Modified in 10.0.35
-parts.partition_alter_myisam         : Include file modified in 10.0.35
-parts.partition_auto_increment_maria : MDEV-14430 - wrong result
-parts.partition_debug_innodb     : MDEV-15095 - table does not exist
-parts.partition_exch_qa_10       : MDEV-11765 - wrong result
+parts.alter_data_directory_innodb      : Added in 10.0.36
+parts.partition_auto_increment_archive : MDEV-16491 - Table marked as crashed
+parts.partition_auto_increment_maria   : MDEV-14430 - wrong result
+parts.partition_exch_qa_10             : MDEV-11765 - wrong result
+parts.truncate_locked                  : Added in 10.0.36
 
 #----------------------------------------------------------------
 
@@ -198,7 +214,8 @@ perfschema_stress.* : MDEV-10996 - tests not maintained
 #----------------------------------------------------------------
 
 plugins.feedback_plugin_send     : MDEV-7932 - ssl failed for url, MDEV-11118 - wrong result
-plugins.server_audit             : MDEV-9562 - crashes on sol10-sparc; modified in 10.0.35
+plugins.processlist              : MDEV-16818 - Syntax error; added in 10.0.36
+plugins.server_audit             : MDEV-9562 - crashes on sol10-sparc; modified in 10.0.36
 plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc, MDEV-14295 - wrong result
 
 #----------------------------------------------------------------
@@ -208,6 +225,7 @@ roles.create_and_grant_role           : MDEV-11772 - wrong result
 #----------------------------------------------------------------
 
 rpl.last_insert_id                    : MDEV-10625 - warnings in error log
+rpl.rename                            : Added in 10.0.36
 rpl.rpl_auto_increment                : MDEV-10417 - Fails on Mips
 rpl.rpl_auto_increment_bug45679       : MDEV-10417 - Fails on Mips
 rpl.rpl_auto_increment_update_failure : MDEV-10625 - warnings in error log
@@ -219,14 +237,17 @@ rpl.rpl_gtid_until                    : MDEV-10625 - warnings in error log
 rpl.rpl_innodb_bug30888               : MDEV-10417 - Fails on Mips
 rpl.rpl_insert                        : MDEV-9329 - Fails on Ubuntu/s390x
 rpl.rpl_insert_delayed                : MDEV-9329 - Fails on Ubuntu/s390x
+rpl.rpl_insert_id_pk                  : MDEV-16567 - Assertion failure
 rpl.rpl_invoked_features              : MDEV-10417 - Fails on Mips
-rpl.rpl_mdev6020                      : MDEV-10630, MDEV-10417 - Timeouts, fails on Mips
+rpl.rpl_mdev6020                      : MDEV-10417 - Timeouts, fails on Mips
+rpl.rpl_mixed_implicit_commit_binlog  : Included file modified in 10.0.36
 rpl.rpl_parallel                      : MDEV-10653 - Timeouts
 rpl.rpl_parallel_mdev6589             : MDEV-12979 - Assertion failure
 rpl.rpl_parallel_multilevel2          : MDEV-14723 - Timeout
 rpl.rpl_parallel_temptable            : MDEV-10356 - Crash in close_thread_tables
 rpl.rpl_partition_innodb              : MDEV-10417 - Fails on Mips
 rpl.rpl_row_basic_11bugs              : MDEV-12171 - Server failed to start
+rpl.rpl_row_implicit_commit_binlog    : Included file modified in 10.0.36
 rpl.rpl_row_index_choice              : MDEV-13409 - Server crash
 rpl.rpl_row_sp001                     : MDEV-9329 - Fails on Ubuntu/s390x
 rpl.rpl_semi_sync                     : MDEV-11220 - Wrong result
@@ -234,6 +255,7 @@ rpl.rpl_semi_sync_uninstall_plugin    : MDEV-7140 - Wrong plugin status
 rpl.rpl_show_slave_hosts              : MDEV-12171 - Server failed to start
 rpl.rpl_skip_replication              : MDEV-9268 - Fails with timeout in sync_slave_with_master on Alpha
 rpl.rpl_slave_grp_exec                : MDEV-10514 - Unexpected deadlock
+rpl.rpl_stm_implicit_commit_binlog    : Included file modified in 10.0.36
 rpl.rpl_sync                          : MDEV-10633 - Database page corruption
 rpl.rpl_temporary_error2              : MDEV-10634 - Wrong number of retries
 
@@ -247,7 +269,7 @@ spider/bg.direct_aggregate      : MDEV-7098 - Trying to unlock mutex that wasn't
 spider/bg.direct_aggregate_part : MDEV-7098 - Trying to unlock mutex that wasn't locked
 spider/bg.spider3_fixes         : MDEV-7098 - Trying to unlock mutex that wasn't locked
 spider/bg.spider_fixes_part     : MDEV-7098 - Trying to unlock mutex that wasn't locked
-spider/bg.ha                    : MDEV-7914 (fixed in 10.3), MDEV-9329 - Crash, failures on s390x
+spider/bg.ha                    : MDEV-7914 (fixed in 10.1), MDEV-9329 - Crash, failures on s390x
 spider/bg.ha_part               : MDEV-9329 - Fails on Ubuntu/s390x
 spider/bg.spider_fixes          : MDEV-7098, MDEV-9329 - Mutex problem, failures on s390x
 spider/bg.vp_fixes              : MDEV-9329 - Fails on Ubuntu/s390x
@@ -267,7 +289,6 @@ stress.ddl_innodb : MDEV-10635 - Testcase timeout
 #----------------------------------------------------------------
 
 sys_vars.autocommit_func2                  : MDEV-9329 - Fails on Ubuntu/s390x
-sys_vars.max_prepared_stmt_count_basic     : Modified in 10.0.35
 sys_vars.thread_cache_size_func            : MDEV-11775 - wrong result
 
 #----------------------------------------------------------------
@@ -310,5 +331,6 @@ unit.my_atomic          : MDEV-15670 - Signal 11 thrown
 
 vcol.not_supported    : MDEV-10639 - Testcase timeout
 vcol.vcol_keys_innodb : MDEV-10639 - Testcase timeout
+vcol.vcol_misc        : MDEV-16651 - Wrong error message; modified in 10.0.36
 
 #----------------------------------------------------------------

From d567f1611e27a17427380e9aae67939792f68ad1 Mon Sep 17 00:00:00 2001
From: Igor Babaev <igor@askmonty.org>
Date: Tue, 24 Jul 2018 20:00:28 -0700
Subject: [PATCH 054/164] MDEV-16820 Lost 'Impossible where' from query with
 inexpensive subquery

This patch fixes another problem introduced by the patch for mdev-4817.
The latter changed Item_cond::fix_fields() in such a way that it could
call the virtual method is_expensive(). With the first its call
the method saves the result in Item::is_expensive_cache. For all next
calls the method returns the result from this cache. So if the item
once was determined as expensive the method always returns true.
For subqueries it's not good, because non-optimized subqueries always
is considered as expensive.
It means that the cache should be invalidated after the call of
optimize_constant_subqueries().
---
 mysql-test/r/subselect.result             | 16 ++++++++++++++++
 mysql-test/r/subselect_no_mat.result      | 16 ++++++++++++++++
 mysql-test/r/subselect_no_opts.result     | 16 ++++++++++++++++
 mysql-test/r/subselect_no_scache.result   | 16 ++++++++++++++++
 mysql-test/r/subselect_no_semijoin.result | 16 ++++++++++++++++
 mysql-test/t/subselect.test               | 15 +++++++++++++++
 sql/item.h                                |  5 +++++
 sql/sql_select.cc                         |  7 +++++++
 8 files changed, 107 insertions(+)

diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result
index e2f2b6521c8..cdedc02f825 100644
--- a/mysql-test/r/subselect.result
+++ b/mysql-test/r/subselect.result
@@ -7171,4 +7171,20 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
diff --git a/mysql-test/r/subselect_no_mat.result b/mysql-test/r/subselect_no_mat.result
index 25ef4a76962..a7291297e7c 100644
--- a/mysql-test/r/subselect_no_mat.result
+++ b/mysql-test/r/subselect_no_mat.result
@@ -7168,6 +7168,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 set optimizer_switch=default;
 select @@optimizer_switch like '%materialization=on%';
diff --git a/mysql-test/r/subselect_no_opts.result b/mysql-test/r/subselect_no_opts.result
index 074874fbd5b..c41fa1be47b 100644
--- a/mysql-test/r/subselect_no_opts.result
+++ b/mysql-test/r/subselect_no_opts.result
@@ -7166,5 +7166,21 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 set @optimizer_switch_for_subselect_test=null;
diff --git a/mysql-test/r/subselect_no_scache.result b/mysql-test/r/subselect_no_scache.result
index de49585b562..1c181357050 100644
--- a/mysql-test/r/subselect_no_scache.result
+++ b/mysql-test/r/subselect_no_scache.result
@@ -7177,6 +7177,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 set optimizer_switch=default;
 select @@optimizer_switch like '%subquery_cache=on%';
diff --git a/mysql-test/r/subselect_no_semijoin.result b/mysql-test/r/subselect_no_semijoin.result
index 46a46c91ddc..89c671252ff 100644
--- a/mysql-test/r/subselect_no_semijoin.result
+++ b/mysql-test/r/subselect_no_semijoin.result
@@ -7166,6 +7166,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 set @optimizer_switch_for_subselect_test=null;
 set @join_cache_level_for_subselect_test=NULL;
diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test
index e6233e9de78..4e35032a789 100644
--- a/mysql-test/t/subselect.test
+++ b/mysql-test/t/subselect.test
@@ -6060,4 +6060,19 @@ and t1.a in (select `test`.`t3`.`c` from `test`.`t3`);
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
 
+--echo #
+--echo # MDEV-16820: impossible where with inexpensive subquery
+--echo #
+
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+
+explain select * from t1 where (select max(b) from t2) = 10;
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+
+drop table t1,t2;
+
 --echo End of 5.5 tests
diff --git a/sql/item.h b/sql/item.h
index d756cf8301b..1bded7377ee 100644
--- a/sql/item.h
+++ b/sql/item.h
@@ -1234,6 +1234,11 @@ public:
   { 
     return FALSE;
   }
+  bool cleanup_is_expensive_cache_processor(uchar *arg)
+  {
+    is_expensive_cache= (int8)(-1);
+    return 0;
+  }
 
   /* To call bool function for all arguments */
   struct bool_func_call_args
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 6a64a0e9952..5228dd4f439 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -1098,6 +1098,13 @@ JOIN::optimize()
   if (optimize_constant_subqueries())
     DBUG_RETURN(1);
 
+  if (conds && conds->has_subquery())
+    (void) conds->walk(&Item::cleanup_is_expensive_cache_processor,
+                       0, (uchar*)0);
+  if (having && having->has_subquery())
+    (void) having->walk(&Item::cleanup_is_expensive_cache_processor,
+                       0, (uchar*)0);
+
   if (setup_jtbm_semi_joins(this, join_list, &conds))
     DBUG_RETURN(1);
 

From 57cde8ccd19675dc98e3cbacf0ef5c72cb188e49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Wed, 25 Jul 2018 08:21:25 +0300
Subject: [PATCH 055/164] MDEV-15822: WSREP: BF lock wait long for trx

In Galera BF (brute force) transactions may not wait for lock requests
and normally BF-transaction would select transaction holding conflicting
locks as a victim for rollback. However, background statistic calculation
transaction is InnoDB internal transaction and it has no thd i.e. it can't be
selected as a victim. If background statistics calculation transaction holds
conflicting locks to statistics tables it will cause BF lock wait long
error message. Correct way to handle background statistics calculation is to
acquire thd for transaction but that change is too big for GA-releases and
there are other reported problems on background statistics calculation.

This fix avoids adding a table to background statistics calculation if
---
 .../r/galera_bf_background_statistics.result  | 95 +++++++++++++++++++
 .../t/galera_bf_background_statistics.opt     |  1 +
 .../t/galera_bf_background_statistics.test    | 49 ++++++++++
 storage/innobase/row/row0mysql.cc             | 25 ++++-
 4 files changed, 167 insertions(+), 3 deletions(-)
 create mode 100644 mysql-test/suite/galera/r/galera_bf_background_statistics.result
 create mode 100644 mysql-test/suite/galera/t/galera_bf_background_statistics.opt
 create mode 100644 mysql-test/suite/galera/t/galera_bf_background_statistics.test

diff --git a/mysql-test/suite/galera/r/galera_bf_background_statistics.result b/mysql-test/suite/galera/r/galera_bf_background_statistics.result
new file mode 100644
index 00000000000..dc367b065f6
--- /dev/null
+++ b/mysql-test/suite/galera/r/galera_bf_background_statistics.result
@@ -0,0 +1,95 @@
+SELECT @@innodb_stats_persistent;
+@@innodb_stats_persistent
+1
+CREATE TABLE t1 (f1 INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, f2 INTEGER DEFAULT NULL) ENGINE=InnoDB;
+INSERT INTO t1(f1) values (NULL);
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+SELECT count(1) from t1;
+count(1)
+16384
+SET AUTOCOMMIT=OFF;
+INSERT INTO t1 VALUES (9999999,NULL);
+SELECT SLEEP(1000);;
+ALTER TABLE t1 CHANGE f2 f2 INTEGER NOT NULL DEFAULT 1;
+Warnings:
+Warning	1265	Data truncated for column 'f2' at row 1
+Warning	1265	Data truncated for column 'f2' at row 2
+Warning	1265	Data truncated for column 'f2' at row 3
+Warning	1265	Data truncated for column 'f2' at row 4
+Warning	1265	Data truncated for column 'f2' at row 5
+Warning	1265	Data truncated for column 'f2' at row 6
+Warning	1265	Data truncated for column 'f2' at row 7
+Warning	1265	Data truncated for column 'f2' at row 8
+Warning	1265	Data truncated for column 'f2' at row 9
+Warning	1265	Data truncated for column 'f2' at row 10
+Warning	1265	Data truncated for column 'f2' at row 11
+Warning	1265	Data truncated for column 'f2' at row 12
+Warning	1265	Data truncated for column 'f2' at row 13
+Warning	1265	Data truncated for column 'f2' at row 14
+Warning	1265	Data truncated for column 'f2' at row 15
+Warning	1265	Data truncated for column 'f2' at row 16
+Warning	1265	Data truncated for column 'f2' at row 17
+Warning	1265	Data truncated for column 'f2' at row 18
+Warning	1265	Data truncated for column 'f2' at row 19
+Warning	1265	Data truncated for column 'f2' at row 20
+Warning	1265	Data truncated for column 'f2' at row 21
+Warning	1265	Data truncated for column 'f2' at row 22
+Warning	1265	Data truncated for column 'f2' at row 23
+Warning	1265	Data truncated for column 'f2' at row 24
+Warning	1265	Data truncated for column 'f2' at row 25
+Warning	1265	Data truncated for column 'f2' at row 26
+Warning	1265	Data truncated for column 'f2' at row 27
+Warning	1265	Data truncated for column 'f2' at row 28
+Warning	1265	Data truncated for column 'f2' at row 29
+Warning	1265	Data truncated for column 'f2' at row 30
+Warning	1265	Data truncated for column 'f2' at row 31
+Warning	1265	Data truncated for column 'f2' at row 32
+Warning	1265	Data truncated for column 'f2' at row 33
+Warning	1265	Data truncated for column 'f2' at row 34
+Warning	1265	Data truncated for column 'f2' at row 35
+Warning	1265	Data truncated for column 'f2' at row 36
+Warning	1265	Data truncated for column 'f2' at row 37
+Warning	1265	Data truncated for column 'f2' at row 38
+Warning	1265	Data truncated for column 'f2' at row 39
+Warning	1265	Data truncated for column 'f2' at row 40
+Warning	1265	Data truncated for column 'f2' at row 41
+Warning	1265	Data truncated for column 'f2' at row 42
+Warning	1265	Data truncated for column 'f2' at row 43
+Warning	1265	Data truncated for column 'f2' at row 44
+Warning	1265	Data truncated for column 'f2' at row 45
+Warning	1265	Data truncated for column 'f2' at row 46
+Warning	1265	Data truncated for column 'f2' at row 47
+Warning	1265	Data truncated for column 'f2' at row 48
+Warning	1265	Data truncated for column 'f2' at row 49
+Warning	1265	Data truncated for column 'f2' at row 50
+Warning	1265	Data truncated for column 'f2' at row 51
+Warning	1265	Data truncated for column 'f2' at row 52
+Warning	1265	Data truncated for column 'f2' at row 53
+Warning	1265	Data truncated for column 'f2' at row 54
+Warning	1265	Data truncated for column 'f2' at row 55
+Warning	1265	Data truncated for column 'f2' at row 56
+Warning	1265	Data truncated for column 'f2' at row 57
+Warning	1265	Data truncated for column 'f2' at row 58
+Warning	1265	Data truncated for column 'f2' at row 59
+Warning	1265	Data truncated for column 'f2' at row 60
+Warning	1265	Data truncated for column 'f2' at row 61
+Warning	1265	Data truncated for column 'f2' at row 62
+Warning	1265	Data truncated for column 'f2' at row 63
+Warning	1265	Data truncated for column 'f2' at row 64
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+wsrep_local_aborts_increment
+1
+DROP TABLE t1;
diff --git a/mysql-test/suite/galera/t/galera_bf_background_statistics.opt b/mysql-test/suite/galera/t/galera_bf_background_statistics.opt
new file mode 100644
index 00000000000..f9b1414a974
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_bf_background_statistics.opt
@@ -0,0 +1 @@
+--innodb_stats_persistent=ON
diff --git a/mysql-test/suite/galera/t/galera_bf_background_statistics.test b/mysql-test/suite/galera/t/galera_bf_background_statistics.test
new file mode 100644
index 00000000000..1030e8d4154
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_bf_background_statistics.test
@@ -0,0 +1,49 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+#
+# Test a local transaction being aborted by a slave one while it is running a SLEEP()
+#
+
+SELECT @@innodb_stats_persistent;
+
+CREATE TABLE t1 (f1 INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, f2 INTEGER DEFAULT NULL) ENGINE=InnoDB;
+INSERT INTO t1(f1) values (NULL);
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+INSERT INTO t1(f1) select NULL from t1;
+SELECT count(1) from t1;
+
+--connection node_2
+SET AUTOCOMMIT=OFF;
+--let $wsrep_local_bf_aborts_before = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_bf_aborts'`
+INSERT INTO t1 VALUES (9999999,NULL);
+--send SELECT SLEEP(1000);
+
+--connection node_1
+ALTER TABLE t1 CHANGE f2 f2 INTEGER NOT NULL DEFAULT 1;
+
+--connection node_2
+--error ER_LOCK_DEADLOCK
+--reap
+
+--let $wsrep_local_bf_aborts_after = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_bf_aborts'`
+
+# Check that wsrep_local_bf_aborts has been incremented by exactly 1
+--disable_query_log
+--eval SELECT $wsrep_local_bf_aborts_after - $wsrep_local_bf_aborts_before = 1 AS wsrep_local_aborts_increment;
+--enable_query_log
+
+DROP TABLE t1;
+
diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc
index 6a971a973f5..963d939380c 100644
--- a/storage/innobase/row/row0mysql.cc
+++ b/storage/innobase/row/row0mysql.cc
@@ -70,6 +70,13 @@ Created 9/17/2000 Heikki Tuuri
 #include "ha_prototypes.h"
 #include <algorithm>
 
+#ifdef WITH_WSREP
+#include "mysql/service_wsrep.h"
+#include "wsrep.h"
+#include "log.h"
+#include "wsrep_mysqld.h"
+#endif
+
 /** Provide optional 4.x backwards compatibility for 5.0 and above */
 UNIV_INTERN ibool	row_rollback_on_timeout	= FALSE;
 
@@ -1093,6 +1100,7 @@ UNIV_INLINE
 void
 row_update_statistics_if_needed(
 /*============================*/
+	trx_t*		trx,
 	dict_table_t*	table)	/*!< in: table */
 {
 	ib_uint64_t	counter;
@@ -1114,6 +1122,16 @@ row_update_statistics_if_needed(
 		if (counter > n_rows / 10 /* 10% */
 		    && dict_stats_auto_recalc_is_enabled(table)) {
 
+#ifdef WITH_WSREP
+			if (wsrep_on(trx->mysql_thd) &&
+			    wsrep_thd_is_BF(trx->mysql_thd, FALSE)) {
+				WSREP_DEBUG("Avoiding background statistics"
+					    " calculation for table %s",
+					    table->name);
+				return;
+			}
+#endif /* WITH_WSREP */
+
 			dict_stats_recalc_pool_add(table);
 			table->stat_modified_counter = 0;
 		}
@@ -1537,7 +1555,8 @@ error_exit:
 		ut_memcpy(prebuilt->row_id, node->row_id_buf, DATA_ROW_ID_LEN);
 	}
 
-	row_update_statistics_if_needed(table);
+	row_update_statistics_if_needed(trx, table);
+
 	trx->op_info = "";
 
 	return(err);
@@ -1921,7 +1940,7 @@ run_again:
 	that changes indexed columns, UPDATEs that change only non-indexed
 	columns would not affect statistics. */
 	if (node->is_delete || !(node->cmpl_info & UPD_NODE_NO_ORD_CHANGE)) {
-		row_update_statistics_if_needed(prebuilt->table);
+		row_update_statistics_if_needed(trx, prebuilt->table);
 	} else {
 		/* Update the table modification counter even when
 		non-indexed columns change if statistics is initialized. */
@@ -2158,7 +2177,7 @@ run_again:
 		}
 	}
 
-	row_update_statistics_if_needed(table);
+	row_update_statistics_if_needed(trx, table);
 
 	return(err);
 }

From c631060713a2af2890284feb7aea96c0cf4ba49f Mon Sep 17 00:00:00 2001
From: Igor Babaev <igor@askmonty.org>
Date: Tue, 24 Jul 2018 23:45:55 -0700
Subject: [PATCH 056/164] MDEV-16820 Lost 'Impossible where' from query with
 inexpensive subquery

This patch fixes another problem introduced by the patch for mdev-4817.
The latter changed Item_cond::fix_fields() in such a way that it could
call the virtual method is_expensive(). With the first its call
the method saves the result in Item::is_expensive_cache. For all next
calls the method returns the result from this cache. So if the item
once was determined as expensive the method always returns true.
For subqueries it's not good, because non-optimized subqueries always
is considered as expensive.
It means that the cache should be invalidated after the call of
optimize_constant_subqueries().
---
 mysql-test/r/subselect.result                 | 16 ++++++++++++++++
 mysql-test/r/subselect_no_exists_to_in.result | 16 ++++++++++++++++
 mysql-test/r/subselect_no_mat.result          | 16 ++++++++++++++++
 mysql-test/r/subselect_no_opts.result         | 16 ++++++++++++++++
 mysql-test/r/subselect_no_scache.result       | 16 ++++++++++++++++
 mysql-test/r/subselect_no_semijoin.result     | 16 ++++++++++++++++
 mysql-test/t/subselect.test                   | 15 +++++++++++++++
 sql/item.h                                    |  5 +++++
 sql/sql_select.cc                             |  7 +++++++
 9 files changed, 123 insertions(+)

diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result
index 79b2810d529..b074fb371a5 100644
--- a/mysql-test/r/subselect.result
+++ b/mysql-test/r/subselect.result
@@ -7211,4 +7211,20 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
diff --git a/mysql-test/r/subselect_no_exists_to_in.result b/mysql-test/r/subselect_no_exists_to_in.result
index 7b2054f4a67..d5aa16a2ce9 100644
--- a/mysql-test/r/subselect_no_exists_to_in.result
+++ b/mysql-test/r/subselect_no_exists_to_in.result
@@ -7211,6 +7211,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 set optimizer_switch=default;
 select @@optimizer_switch like '%exists_to_in=off%';
diff --git a/mysql-test/r/subselect_no_mat.result b/mysql-test/r/subselect_no_mat.result
index 51c5263ad8a..aff68bd6729 100644
--- a/mysql-test/r/subselect_no_mat.result
+++ b/mysql-test/r/subselect_no_mat.result
@@ -7204,6 +7204,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 set optimizer_switch=default;
 select @@optimizer_switch like '%materialization=on%';
diff --git a/mysql-test/r/subselect_no_opts.result b/mysql-test/r/subselect_no_opts.result
index 574af7b4d1f..f1181785a5c 100644
--- a/mysql-test/r/subselect_no_opts.result
+++ b/mysql-test/r/subselect_no_opts.result
@@ -7202,5 +7202,21 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 set @optimizer_switch_for_subselect_test=null;
diff --git a/mysql-test/r/subselect_no_scache.result b/mysql-test/r/subselect_no_scache.result
index 03285cf2968..6cefce21c20 100644
--- a/mysql-test/r/subselect_no_scache.result
+++ b/mysql-test/r/subselect_no_scache.result
@@ -7217,6 +7217,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 set optimizer_switch=default;
 select @@optimizer_switch like '%subquery_cache=on%';
diff --git a/mysql-test/r/subselect_no_semijoin.result b/mysql-test/r/subselect_no_semijoin.result
index 6669c27ede7..884374a74b1 100644
--- a/mysql-test/r/subselect_no_semijoin.result
+++ b/mysql-test/r/subselect_no_semijoin.result
@@ -7202,6 +7202,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 set @optimizer_switch_for_subselect_test=null;
 set @join_cache_level_for_subselect_test=NULL;
diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test
index 8c9a4893be2..390d36361b2 100644
--- a/mysql-test/t/subselect.test
+++ b/mysql-test/t/subselect.test
@@ -6096,4 +6096,19 @@ and t1.a in (select `test`.`t3`.`c` from `test`.`t3`);
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
 
+--echo #
+--echo # MDEV-16820: impossible where with inexpensive subquery
+--echo #
+
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+
+explain select * from t1 where (select max(b) from t2) = 10;
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+
+drop table t1,t2;
+
 --echo End of 5.5 tests
diff --git a/sql/item.h b/sql/item.h
index b111d74057c..7c95239b242 100644
--- a/sql/item.h
+++ b/sql/item.h
@@ -1275,6 +1275,11 @@ public:
   virtual bool exists2in_processor(uchar *opt_arg) { return 0; }
   virtual bool find_selective_predicates_list_processor(uchar *opt_arg)
   { return 0; }
+  bool cleanup_is_expensive_cache_processor(uchar *arg)
+  {
+    is_expensive_cache= (int8)(-1);
+    return 0;
+  }
 
   /* To call bool function for all arguments */
   struct bool_func_call_args
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 43b8d2b8636..19a727f696b 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -1207,6 +1207,13 @@ TODO: make view to decide if it is possible to write to WHERE directly or make S
   if (optimize_constant_subqueries())
     DBUG_RETURN(1);
 
+  if (conds && conds->has_subquery())
+    (void) conds->walk(&Item::cleanup_is_expensive_cache_processor,
+                       0, (uchar*)0);
+  if (having && having->has_subquery())
+    (void) having->walk(&Item::cleanup_is_expensive_cache_processor,
+                       0, (uchar*)0);
+
   if (setup_jtbm_semi_joins(this, join_list, &conds))
     DBUG_RETURN(1);
 

From 1fde449f1d4e8ebe12e989754bfdc84b123cf0b2 Mon Sep 17 00:00:00 2001
From: Igor Babaev <igor@askmonty.org>
Date: Tue, 24 Jul 2018 23:45:55 -0700
Subject: [PATCH 057/164] MDEV-16820 Lost 'Impossible where' from query with
 inexpensive subquery

This patch fixes another problem introduced by the patch for mdev-4817.
The latter changed Item_cond::fix_fields() in such a way that it could
call the virtual method is_expensive(). With the first its call
the method saves the result in Item::is_expensive_cache. For all next
calls the method returns the result from this cache. So if the item
once was determined as expensive the method always returns true.
For subqueries it's not good, because non-optimized subqueries always
is considered as expensive.
It means that the cache should be invalidated after the call of
optimize_constant_subqueries().
---
 mysql-test/r/subselect.result                 | 16 ++++++++++++++++
 mysql-test/r/subselect_no_exists_to_in.result | 16 ++++++++++++++++
 mysql-test/r/subselect_no_mat.result          | 16 ++++++++++++++++
 mysql-test/r/subselect_no_opts.result         | 16 ++++++++++++++++
 mysql-test/r/subselect_no_scache.result       | 16 ++++++++++++++++
 mysql-test/r/subselect_no_semijoin.result     | 16 ++++++++++++++++
 mysql-test/t/subselect.test                   | 15 +++++++++++++++
 sql/item.h                                    |  5 +++++
 sql/sql_select.cc                             |  7 +++++++
 9 files changed, 123 insertions(+)

diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result
index 40f936fb3b4..69db0959373 100644
--- a/mysql-test/r/subselect.result
+++ b/mysql-test/r/subselect.result
@@ -7235,6 +7235,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/subselect_no_exists_to_in.result b/mysql-test/r/subselect_no_exists_to_in.result
index a23820820ef..ef3bf7ef692 100644
--- a/mysql-test/r/subselect_no_exists_to_in.result
+++ b/mysql-test/r/subselect_no_exists_to_in.result
@@ -7235,6 +7235,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/subselect_no_mat.result b/mysql-test/r/subselect_no_mat.result
index baa74307f89..703572e0ef8 100644
--- a/mysql-test/r/subselect_no_mat.result
+++ b/mysql-test/r/subselect_no_mat.result
@@ -7228,6 +7228,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/subselect_no_opts.result b/mysql-test/r/subselect_no_opts.result
index 039f2fe1a9e..39a098b754b 100644
--- a/mysql-test/r/subselect_no_opts.result
+++ b/mysql-test/r/subselect_no_opts.result
@@ -7226,6 +7226,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/subselect_no_scache.result b/mysql-test/r/subselect_no_scache.result
index 0ce77bbb376..b9a59bf3952 100644
--- a/mysql-test/r/subselect_no_scache.result
+++ b/mysql-test/r/subselect_no_scache.result
@@ -7241,6 +7241,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/subselect_no_semijoin.result b/mysql-test/r/subselect_no_semijoin.result
index 574e78122f1..f1df85fed6e 100644
--- a/mysql-test/r/subselect_no_semijoin.result
+++ b/mysql-test/r/subselect_no_semijoin.result
@@ -7226,6 +7226,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 # End of 10.0 tests
 #
diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test
index 717871db61d..a4c4c21610e 100644
--- a/mysql-test/t/subselect.test
+++ b/mysql-test/t/subselect.test
@@ -6098,6 +6098,21 @@ and t1.a in (select `test`.`t3`.`c` from `test`.`t3`);
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
 
+--echo #
+--echo # MDEV-16820: impossible where with inexpensive subquery
+--echo #
+
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+
+explain select * from t1 where (select max(b) from t2) = 10;
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+
+drop table t1,t2;
+
 --echo End of 5.5 tests
 --echo # End of 10.0 tests
 
diff --git a/sql/item.h b/sql/item.h
index 139ce40b25b..dcd773ffc59 100644
--- a/sql/item.h
+++ b/sql/item.h
@@ -1449,6 +1449,11 @@ public:
   virtual bool exists2in_processor(uchar *opt_arg) { return 0; }
   virtual bool find_selective_predicates_list_processor(uchar *opt_arg)
   { return 0; }
+  bool cleanup_is_expensive_cache_processor(uchar *arg)
+  {
+    is_expensive_cache= (int8)(-1);
+    return 0;
+  }
 
   /* To call bool function for all arguments */
   struct bool_func_call_args
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index e3b725ce1f6..460a64e1a7a 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -1212,6 +1212,13 @@ JOIN::optimize_inner()
   if (optimize_constant_subqueries())
     DBUG_RETURN(1);
 
+  if (conds && conds->has_subquery())
+    (void) conds->walk(&Item::cleanup_is_expensive_cache_processor,
+                       0, (uchar*)0);
+  if (having && having->has_subquery())
+    (void) having->walk(&Item::cleanup_is_expensive_cache_processor,
+                       0, (uchar*)0);
+
   if (setup_jtbm_semi_joins(this, join_list, &conds))
     DBUG_RETURN(1);
 

From 969939e89c29bb3f3d91f8b4ab539601b5daa4d8 Mon Sep 17 00:00:00 2001
From: Sachin <sachin.setiya@mariadb.com>
Date: Wed, 25 Jul 2018 12:54:37 +0530
Subject: [PATCH 058/164] MDEV-16821 Set password for user makes rpl test to
 fail

Actually if we use "set password for " command this changes the checksum
of mysql.user table
-localhost      root            Y       Y       Y       Y       Y       Y       Y       Y     YY       Y       Y       Y       Y       Y       Y       Y       Y       Y       Y       Y     $
Y       Y       Y       Y       Y       Y       Y                                       0     00       0                       N       N               0.000000
+localhost      root            Y       Y       Y       Y       Y       Y       Y       Y     YY       Y       Y       Y       Y       Y       Y       Y       Y       Y       Y       Y     Y
Y       Y       Y       Y       Y       Y       Y                                       0     00       0       mysql_native_password           N       N               0.000000
In short we replace '' with mysql_native_password
which make checksum to be different, and hence check test case fails.

So we use UPDATE mysql.user command.
---
 mysql-test/extra/rpl_tests/rpl_row_001.test  | 4 ++--
 mysql-test/suite/rpl/r/rpl_row_001.result    | 4 ++--
 mysql-test/suite/rpl/r/rpl_stm_000001.result | 4 ++--
 mysql-test/suite/rpl/t/rpl_stm_000001.test   | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/mysql-test/extra/rpl_tests/rpl_row_001.test b/mysql-test/extra/rpl_tests/rpl_row_001.test
index 8eb684e0dff..2706336bdea 100644
--- a/mysql-test/extra/rpl_tests/rpl_row_001.test
+++ b/mysql-test/extra/rpl_tests/rpl_row_001.test
@@ -14,7 +14,7 @@ connection slave;
 sync_with_master;
 STOP SLAVE;
 connection master;
-SET PASSWORD FOR root@"localhost" = PASSWORD('foo');
+UPDATE mysql.user SET password=password('foo') WHERE host='localhost' AND user='root';
 connection slave;
 START SLAVE;
 connection master;
@@ -22,7 +22,7 @@ connection master;
 # Give slave time to do at last one failed connect retry
 # This one must be short so that the slave will not stop retrying
 real_sleep 2;
-SET PASSWORD FOR root@"localhost" = PASSWORD('');
+UPDATE mysql.user SET password=password('') WHERE host='localhost' AND user='root';
 # Give slave time to connect (will retry every second)
 sleep 2;
 
diff --git a/mysql-test/suite/rpl/r/rpl_row_001.result b/mysql-test/suite/rpl/r/rpl_row_001.result
index 94985548f59..3ca694a8e76 100644
--- a/mysql-test/suite/rpl/r/rpl_row_001.result
+++ b/mysql-test/suite/rpl/r/rpl_row_001.result
@@ -18,11 +18,11 @@ Ababa
 connection slave;
 STOP SLAVE;
 connection master;
-SET PASSWORD FOR root@"localhost" = PASSWORD('foo');
+UPDATE mysql.user SET password=password('foo') WHERE host='localhost' AND user='root';
 connection slave;
 START SLAVE;
 connection master;
-SET PASSWORD FOR root@"localhost" = PASSWORD('');
+UPDATE mysql.user SET password=password('') WHERE host='localhost' AND user='root';
 CREATE TABLE t3(n INT);
 INSERT INTO t3 VALUES(1),(2);
 connection slave;
diff --git a/mysql-test/suite/rpl/r/rpl_stm_000001.result b/mysql-test/suite/rpl/r/rpl_stm_000001.result
index 5680d22f233..0b9ed6fc09c 100644
--- a/mysql-test/suite/rpl/r/rpl_stm_000001.result
+++ b/mysql-test/suite/rpl/r/rpl_stm_000001.result
@@ -19,11 +19,11 @@ abandons
 connection slave;
 stop slave;
 connection master;
-set password for root@"localhost" = password('foo');
+UPDATE mysql.user SET password=password('foo') WHERE host='localhost' AND user='root';
 connection slave;
 start slave;
 connection master;
-set password for root@"localhost" = password('');
+UPDATE mysql.user SET password=password('') WHERE host='localhost' AND user='root';
 create table t3(n int);
 insert into t3 values(1),(2);
 connection slave;
diff --git a/mysql-test/suite/rpl/t/rpl_stm_000001.test b/mysql-test/suite/rpl/t/rpl_stm_000001.test
index 7e8f669ff34..119fd6168e0 100644
--- a/mysql-test/suite/rpl/t/rpl_stm_000001.test
+++ b/mysql-test/suite/rpl/t/rpl_stm_000001.test
@@ -19,7 +19,7 @@ select * from t1 limit 10;
 sync_slave_with_master;
 stop slave;
 connection master;
-set password for root@"localhost" = password('foo');
+UPDATE mysql.user SET password=password('foo') WHERE host='localhost' AND user='root';
 connection slave;
 start slave;
 connection master;
@@ -27,7 +27,7 @@ connection master;
 # Give slave time to do at last one failed connect retry
 # This one must be short so that the slave will not stop retrying
 real_sleep 2;
-set password for root@"localhost" = password('');
+UPDATE mysql.user SET password=password('') WHERE host='localhost' AND user='root';
 # Give slave time to connect (will retry every second)
 sleep 2;
 

From de85355436e483902e0fff432348bc16f9ec1557 Mon Sep 17 00:00:00 2001
From: Thirunarayanan Balathandayuthapani <thiru@mariadb.com>
Date: Wed, 25 Jul 2018 13:56:39 +0530
Subject: [PATCH 059/164] MDEV-16713	Hangs server with repeating log entry

	At most one transaction can be active at a time for temporary
tables. There is no need to check previous version of record for the
temporary tables.
---
 .../suite/innodb/r/temporary_table.result       | 13 +++++++++++++
 mysql-test/suite/innodb/t/temporary_table.test  | 17 +++++++++++++++++
 storage/innobase/row/row0sel.cc                 |  7 +++++++
 storage/innobase/row/row0vers.cc                |  2 ++
 4 files changed, 39 insertions(+)

diff --git a/mysql-test/suite/innodb/r/temporary_table.result b/mysql-test/suite/innodb/r/temporary_table.result
index 64eb3270934..628bb648ead 100644
--- a/mysql-test/suite/innodb/r/temporary_table.result
+++ b/mysql-test/suite/innodb/r/temporary_table.result
@@ -650,3 +650,16 @@ SELECT * FROM t1;
 f1
 0
 DROP TABLE t1;
+create procedure t1_proc()
+begin
+DECLARE var INT UNSIGNED;
+CREATE TEMPORARY TABLE t1(f1 INT UNSIGNED, f2 INT UNSIGNED, KEY( f1, f2 ) )engine=innodb;
+SET TRANSACTION ISOLATION LEVEL READ COMMITTED;
+START TRANSACTION;
+INSERT INTO t1 SET f1 = 1, f2 = 1;
+UPDATE t1 SET f2 = 2;
+SET var = ( SELECT 1 FROM t1 );
+DROP TABLE t1;
+END//
+call t1_proc;
+drop procedure t1_proc;
diff --git a/mysql-test/suite/innodb/t/temporary_table.test b/mysql-test/suite/innodb/t/temporary_table.test
index db87367dbb9..6856ca86323 100644
--- a/mysql-test/suite/innodb/t/temporary_table.test
+++ b/mysql-test/suite/innodb/t/temporary_table.test
@@ -476,3 +476,20 @@ UPDATE t1 SET f1 = 0;
 ROLLBACK;
 SELECT * FROM t1;
 DROP TABLE t1;
+
+delimiter //;
+create procedure t1_proc()
+begin
+DECLARE var INT UNSIGNED;
+CREATE TEMPORARY TABLE t1(f1 INT UNSIGNED, f2 INT UNSIGNED, KEY( f1, f2 ) )engine=innodb;
+SET TRANSACTION ISOLATION LEVEL READ COMMITTED;
+START TRANSACTION;
+INSERT INTO t1 SET f1 = 1, f2 = 1;
+UPDATE t1 SET f2 = 2;
+SET var = ( SELECT 1 FROM t1 );
+DROP TABLE t1;
+END//
+delimiter ;//
+
+call t1_proc;
+drop procedure t1_proc;
diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc
index d21fd4a10ed..5d142c24490 100644
--- a/storage/innobase/row/row0sel.cc
+++ b/storage/innobase/row/row0sel.cc
@@ -5023,6 +5023,13 @@ wrong_offs:
 			if (!rec_get_deleted_flag(rec, comp)) {
 				goto no_gap_lock;
 			}
+
+			/* At most one transaction can be active
+			for temporary table. */
+			if (dict_table_is_temporary(clust_index->table)) {
+				goto no_gap_lock;
+			}
+
 			if (index == clust_index) {
 				trx_id_t trx_id = row_get_rec_trx_id(
 					rec, index, offsets);
diff --git a/storage/innobase/row/row0vers.cc b/storage/innobase/row/row0vers.cc
index f58f0a47bd5..24acd04c84b 100644
--- a/storage/innobase/row/row0vers.cc
+++ b/storage/innobase/row/row0vers.cc
@@ -114,6 +114,8 @@ row_vers_impl_x_locked_low(
 	trx_id = row_get_rec_trx_id(clust_rec, clust_index, clust_offsets);
 	corrupt = FALSE;
 
+	ut_ad(!dict_table_is_temporary(clust_index->table));
+
 	trx_t*	trx = trx_rw_is_active(trx_id, &corrupt, true);
 
 	if (trx == 0) {

From f9b43c2565284feefe94e4feaa4c02bf25a6f921 Mon Sep 17 00:00:00 2001
From: Varun Gupta <varun.gupta@mariadb.com>
Date: Wed, 25 Jul 2018 14:20:16 +0530
Subject: [PATCH 060/164] MDEV-16751: Server crashes in st_join_table::cleanup
 or TABLE_LIST::is_with_table_recursive_reference             with
 join_cache_level>2

During muliple equality propagation for a query in which we have an IN subquery, the items in the select list of the
subquery may not be part of the multiple equality because there might be another occurence of the same field in the
where clause of the subquery.
So we keyuse_is_valid_for_access_in_chosen_plan function which expects the items in the select list of the subquery to
be same to the ones in the multiple equality (through these multiple equalities we create keyuse array).
The solution would be that we expect the same field not the same Item because when we have SEMI JOIN MATERIALIZATION SCAN,
we use copy back technique to copies back the materialised table fields to the original fields of the base tables.
---
 mysql-test/r/subselect_mat.result    | 35 ++++++++++++++++++++++++++++
 mysql-test/r/subselect_sj_mat.result | 35 ++++++++++++++++++++++++++++
 mysql-test/t/subselect_sj_mat.test   | 25 ++++++++++++++++++++
 sql/opt_subselect.cc                 |  3 +++
 sql/sql_select.cc                    |  9 +++++--
 5 files changed, 105 insertions(+), 2 deletions(-)

diff --git a/mysql-test/r/subselect_mat.result b/mysql-test/r/subselect_mat.result
index 00448ac4f91..a56d076d528 100644
--- a/mysql-test/r/subselect_mat.result
+++ b/mysql-test/r/subselect_mat.result
@@ -2360,6 +2360,41 @@ ec70316637232000158bbfc8bcbe5d60
 ebb4620037332000158bbfc8bcbe5d89
 DROP TABLE t1,t2,t3;
 set optimizer_switch=@save_optimizer_switch;
+#
+# MDEV-16751: Server crashes in st_join_table::cleanup or 
+# TABLE_LIST::is_with_table_recursive_reference with join_cache_level>2
+#
+set @save_join_cache_level= @@join_cache_level;
+set join_cache_level=4;
+CREATE TABLE t1 ( id int NOT NULL);
+INSERT INTO t1 VALUES (11),(12),(13),(14),(15),(16),(17),(18),(19);
+CREATE TABLE t2 (i1 int NOT NULL, i2 int NOT NULL) ;
+INSERT INTO t2 VALUES  (11,11),(12,12),(13,13);
+explain
+SELECT 1 FROM t1 where t1.id IN (SELECT t2.i1 FROM t2 WHERE t2.i1 = t2.i2);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	<subquery2>	ALL	distinct_key	NULL	NULL	NULL	3	
+1	PRIMARY	t1	hash_ALL	NULL	#hash#$hj	4	test.t2.i1	9	Using where; Using join buffer (flat, BNLH join)
+2	MATERIALIZED	t2	ALL	NULL	NULL	NULL	NULL	3	Using where
+SELECT 1 FROM t1 where t1.id IN (SELECT t2.i1 FROM t2 WHERE t2.i1 = t2.i2);
+1
+1
+1
+1
+set @@join_cache_level= @save_join_cache_level;
+alter table t1 add key(id);
+explain
+SELECT 1 FROM t1 where t1.id IN (SELECT t2.i1 FROM t2 WHERE t2.i1 = t2.i2);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	<subquery2>	ALL	distinct_key	NULL	NULL	NULL	3	
+1	PRIMARY	t1	ref	id	id	4	test.t2.i1	2	Using index
+2	MATERIALIZED	t2	ALL	NULL	NULL	NULL	NULL	3	Using where
+SELECT 1 FROM t1 where t1.id IN (SELECT t2.i1 FROM t2 WHERE t2.i1 = t2.i2);
+1
+1
+1
+1
+drop table t1,t2;
 # End of 5.5 tests
 set @subselect_mat_test_optimizer_switch_value=null;
 set @@optimizer_switch='materialization=on,in_to_exists=off,semijoin=off';
diff --git a/mysql-test/r/subselect_sj_mat.result b/mysql-test/r/subselect_sj_mat.result
index cb5012a91c9..fa62259e3d6 100644
--- a/mysql-test/r/subselect_sj_mat.result
+++ b/mysql-test/r/subselect_sj_mat.result
@@ -2400,4 +2400,39 @@ ec70316637232000158bbfc8bcbe5d60
 ebb4620037332000158bbfc8bcbe5d89
 DROP TABLE t1,t2,t3;
 set optimizer_switch=@save_optimizer_switch;
+#
+# MDEV-16751: Server crashes in st_join_table::cleanup or 
+# TABLE_LIST::is_with_table_recursive_reference with join_cache_level>2
+#
+set @save_join_cache_level= @@join_cache_level;
+set join_cache_level=4;
+CREATE TABLE t1 ( id int NOT NULL);
+INSERT INTO t1 VALUES (11),(12),(13),(14),(15),(16),(17),(18),(19);
+CREATE TABLE t2 (i1 int NOT NULL, i2 int NOT NULL) ;
+INSERT INTO t2 VALUES  (11,11),(12,12),(13,13);
+explain
+SELECT 1 FROM t1 where t1.id IN (SELECT t2.i1 FROM t2 WHERE t2.i1 = t2.i2);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	<subquery2>	ALL	distinct_key	NULL	NULL	NULL	3	
+1	PRIMARY	t1	hash_ALL	NULL	#hash#$hj	4	test.t2.i1	9	Using where; Using join buffer (flat, BNLH join)
+2	MATERIALIZED	t2	ALL	NULL	NULL	NULL	NULL	3	Using where
+SELECT 1 FROM t1 where t1.id IN (SELECT t2.i1 FROM t2 WHERE t2.i1 = t2.i2);
+1
+1
+1
+1
+set @@join_cache_level= @save_join_cache_level;
+alter table t1 add key(id);
+explain
+SELECT 1 FROM t1 where t1.id IN (SELECT t2.i1 FROM t2 WHERE t2.i1 = t2.i2);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	<subquery2>	ALL	distinct_key	NULL	NULL	NULL	3	
+1	PRIMARY	t1	ref	id	id	4	test.t2.i1	2	Using index
+2	MATERIALIZED	t2	ALL	NULL	NULL	NULL	NULL	3	Using where
+SELECT 1 FROM t1 where t1.id IN (SELECT t2.i1 FROM t2 WHERE t2.i1 = t2.i2);
+1
+1
+1
+1
+drop table t1,t2;
 # End of 5.5 tests
diff --git a/mysql-test/t/subselect_sj_mat.test b/mysql-test/t/subselect_sj_mat.test
index 90f63bea561..f1b64337702 100644
--- a/mysql-test/t/subselect_sj_mat.test
+++ b/mysql-test/t/subselect_sj_mat.test
@@ -2151,4 +2151,29 @@ eval $q;
 DROP TABLE t1,t2,t3;
 set optimizer_switch=@save_optimizer_switch;
 
+--echo #
+--echo # MDEV-16751: Server crashes in st_join_table::cleanup or 
+--echo # TABLE_LIST::is_with_table_recursive_reference with join_cache_level>2
+--echo #
+
+set @save_join_cache_level= @@join_cache_level;
+set join_cache_level=4;
+CREATE TABLE t1 ( id int NOT NULL);
+INSERT INTO t1 VALUES (11),(12),(13),(14),(15),(16),(17),(18),(19);
+ 
+CREATE TABLE t2 (i1 int NOT NULL, i2 int NOT NULL) ;
+INSERT INTO t2 VALUES  (11,11),(12,12),(13,13);
+
+explain
+SELECT 1 FROM t1 where t1.id IN (SELECT t2.i1 FROM t2 WHERE t2.i1 = t2.i2);
+SELECT 1 FROM t1 where t1.id IN (SELECT t2.i1 FROM t2 WHERE t2.i1 = t2.i2);
+
+set @@join_cache_level= @save_join_cache_level;
+alter table t1 add key(id);
+
+explain
+SELECT 1 FROM t1 where t1.id IN (SELECT t2.i1 FROM t2 WHERE t2.i1 = t2.i2);
+SELECT 1 FROM t1 where t1.id IN (SELECT t2.i1 FROM t2 WHERE t2.i1 = t2.i2);
+
+drop table t1,t2;
 --echo # End of 5.5 tests
diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc
index 14b2aaee591..783e3d4d22f 100644
--- a/sql/opt_subselect.cc
+++ b/sql/opt_subselect.cc
@@ -815,6 +815,9 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
           details)
         * require that compared columns have exactly the same type. This is
           a temporary measure to avoid BUG#36752-type problems.
+
+    JOIN_TAB::keyuse_is_valid_for_access_in_chosen_plan expects that for Semi Join Materialization
+    Scan all the items in the select list of the IN Subquery are of the type Item::FIELD_ITEM.
 */
 
 static 
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 5228dd4f439..45a1372988b 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -7445,8 +7445,13 @@ bool JOIN_TAB::keyuse_is_valid_for_access_in_chosen_plan(JOIN *join,
   st_select_lex *sjm_sel= emb_sj_nest->sj_subq_pred->unit->first_select(); 
   for (uint i= 0; i < sjm_sel->item_list.elements; i++)
   {
-    if (sjm_sel->ref_pointer_array[i] == keyuse->val)
-      return true;
+    DBUG_ASSERT(sjm_sel->ref_pointer_array[i]->type() == Item::FIELD_ITEM);
+    if (keyuse->val->type() == Item::FIELD_ITEM)
+    {
+      Field *field = ((Item_field*)sjm_sel->ref_pointer_array[i])->field;
+      if (field->eq(((Item_field*)keyuse->val)->field))
+        return true;
+    }
   }
   return false; 
 }

From 37dee22d27c55c14f0be7005c3e5bdbb09e1fc92 Mon Sep 17 00:00:00 2001
From: Varun Gupta <varunraiko1803@gmail.com>
Date: Wed, 25 Jul 2018 21:17:50 +0530
Subject: [PATCH 061/164] MDEV-15454: Nested SELECT IN returns wrong results

In this case we are setting the field Item_func_eq::in_eqaulity_no for the semi-join equalities.
This helps us to remove these equalites as the inner tables are not available during parent select execution
while the outer tables are not available during materialization phase.
We only have it set for the equalites for the fields involved with the IN subquery
and reset it for the equalities which do not belong to the IN subquery.

For example in case of nested IN subqueries:

    SELECT t1.a FROM t1 WHERE t1.a IN
      (SELECT t2.a FROM t2 where t2.b IN
          (select t3.b from t3 where t3.c=27 ))

there are two equalites involving the fields of the IN subquery

1) t2.b = t3.b :  the field Item_func_eq::in_eqaulity_no is set when we merge the grandchild select into the child select
2) t1.a = t2.a :  the field Item_func_eq::in_eqaulity_no is set when we merge the child select into the parent select

But when we perform case 2) we should ensure that we reset the equalities in the child's WHERE clause.
---
 mysql-test/r/subselect_mat.result    | 41 ++++++++++++++++++
 mysql-test/r/subselect_sj_mat.result | 41 ++++++++++++++++++
 mysql-test/t/subselect_sj_mat.test   | 45 ++++++++++++++++++++
 sql/opt_subselect.cc                 | 63 ++++++++++++++++++++++++++++
 4 files changed, 190 insertions(+)

diff --git a/mysql-test/r/subselect_mat.result b/mysql-test/r/subselect_mat.result
index a56d076d528..7f39f1fb788 100644
--- a/mysql-test/r/subselect_mat.result
+++ b/mysql-test/r/subselect_mat.result
@@ -2395,6 +2395,47 @@ SELECT 1 FROM t1 where t1.id IN (SELECT t2.i1 FROM t2 WHERE t2.i1 = t2.i2);
 1
 1
 drop table t1,t2;
+#
+# MDEV-15454: Nested SELECT IN returns wrong results
+#
+CREATE TABLE t1 ( a int NOT NULL PRIMARY KEY);
+CREATE TABLE t2 ( a int, b int );
+INSERT INTO t2  VALUES (7878, 96),(3465, 96),(1403, 96),(4189, 96),(8732, 96), (5,96);
+CREATE TABLE t3 (c int unsigned NOT NULL, b int unsigned NOT NULL, PRIMARY KEY (c,b));
+INSERT INTO t3 (c, b) VALUES (27, 96);
+CREATE PROCEDURE prepare_data()
+BEGIN
+DECLARE i INT DEFAULT 1;
+WHILE i < 1000 DO
+INSERT INTO t1 (a) VALUES (i);
+INSERT INTO t2 (a,b) VALUES (i,56);
+INSERT INTO t3 (c,b) VALUES (i,i);
+SET i = i + 1;
+END WHILE;
+END$$
+CALL prepare_data();
+SELECT t2.a FROM t2 WHERE t2.b IN (SELECT t3.b FROM t3 WHERE t3.c= 27);
+a
+7878
+3465
+1403
+4189
+8732
+5
+set @save_optimizer_switch= @@optimizer_switch;
+SET optimizer_switch='materialization=off';
+SELECT t1.a FROM t1
+WHERE t1.a IN (SELECT t2.a FROM t2 WHERE t2.b IN (SELECT t3.b FROM t3 WHERE t3.c= 27)) LIMIT 5;
+a
+5
+SET optimizer_switch='materialization=on';
+SELECT t1.a FROM t1
+WHERE t1.a IN (SELECT t2.a FROM t2 WHERE t2.b IN (SELECT t3.b FROM t3 WHERE t3.c= 27)) LIMIT 5;
+a
+5
+drop procedure prepare_data;
+set @@optimizer_switch= @save_optimizer_switch;
+drop table t1,t2,t3;
 # End of 5.5 tests
 set @subselect_mat_test_optimizer_switch_value=null;
 set @@optimizer_switch='materialization=on,in_to_exists=off,semijoin=off';
diff --git a/mysql-test/r/subselect_sj_mat.result b/mysql-test/r/subselect_sj_mat.result
index fa62259e3d6..44006eadfc6 100644
--- a/mysql-test/r/subselect_sj_mat.result
+++ b/mysql-test/r/subselect_sj_mat.result
@@ -2435,4 +2435,45 @@ SELECT 1 FROM t1 where t1.id IN (SELECT t2.i1 FROM t2 WHERE t2.i1 = t2.i2);
 1
 1
 drop table t1,t2;
+#
+# MDEV-15454: Nested SELECT IN returns wrong results
+#
+CREATE TABLE t1 ( a int NOT NULL PRIMARY KEY);
+CREATE TABLE t2 ( a int, b int );
+INSERT INTO t2  VALUES (7878, 96),(3465, 96),(1403, 96),(4189, 96),(8732, 96), (5,96);
+CREATE TABLE t3 (c int unsigned NOT NULL, b int unsigned NOT NULL, PRIMARY KEY (c,b));
+INSERT INTO t3 (c, b) VALUES (27, 96);
+CREATE PROCEDURE prepare_data()
+BEGIN
+DECLARE i INT DEFAULT 1;
+WHILE i < 1000 DO
+INSERT INTO t1 (a) VALUES (i);
+INSERT INTO t2 (a,b) VALUES (i,56);
+INSERT INTO t3 (c,b) VALUES (i,i);
+SET i = i + 1;
+END WHILE;
+END$$
+CALL prepare_data();
+SELECT t2.a FROM t2 WHERE t2.b IN (SELECT t3.b FROM t3 WHERE t3.c= 27);
+a
+7878
+3465
+1403
+4189
+8732
+5
+set @save_optimizer_switch= @@optimizer_switch;
+SET optimizer_switch='materialization=off';
+SELECT t1.a FROM t1
+WHERE t1.a IN (SELECT t2.a FROM t2 WHERE t2.b IN (SELECT t3.b FROM t3 WHERE t3.c= 27)) LIMIT 5;
+a
+5
+SET optimizer_switch='materialization=on';
+SELECT t1.a FROM t1
+WHERE t1.a IN (SELECT t2.a FROM t2 WHERE t2.b IN (SELECT t3.b FROM t3 WHERE t3.c= 27)) LIMIT 5;
+a
+5
+drop procedure prepare_data;
+set @@optimizer_switch= @save_optimizer_switch;
+drop table t1,t2,t3;
 # End of 5.5 tests
diff --git a/mysql-test/t/subselect_sj_mat.test b/mysql-test/t/subselect_sj_mat.test
index f1b64337702..794545b53a7 100644
--- a/mysql-test/t/subselect_sj_mat.test
+++ b/mysql-test/t/subselect_sj_mat.test
@@ -2176,4 +2176,49 @@ SELECT 1 FROM t1 where t1.id IN (SELECT t2.i1 FROM t2 WHERE t2.i1 = t2.i2);
 SELECT 1 FROM t1 where t1.id IN (SELECT t2.i1 FROM t2 WHERE t2.i1 = t2.i2);
 
 drop table t1,t2;
+
+--echo #
+--echo # MDEV-15454: Nested SELECT IN returns wrong results
+--echo #
+
+CREATE TABLE t1 ( a int NOT NULL PRIMARY KEY);
+
+CREATE TABLE t2 ( a int, b int );
+INSERT INTO t2  VALUES (7878, 96),(3465, 96),(1403, 96),(4189, 96),(8732, 96), (5,96);
+
+CREATE TABLE t3 (c int unsigned NOT NULL, b int unsigned NOT NULL, PRIMARY KEY (c,b));
+INSERT INTO t3 (c, b) VALUES (27, 96);
+
+DELIMITER $$;
+CREATE PROCEDURE prepare_data()
+BEGIN
+  DECLARE i INT DEFAULT 1;
+  WHILE i < 1000 DO
+    INSERT INTO t1 (a) VALUES (i);
+    INSERT INTO t2 (a,b) VALUES (i,56);
+    INSERT INTO t3 (c,b) VALUES (i,i);
+    SET i = i + 1;
+  END WHILE;
+END$$
+DELIMITER ;$$
+
+CALL prepare_data();
+
+SELECT t2.a FROM t2 WHERE t2.b IN (SELECT t3.b FROM t3 WHERE t3.c= 27);
+
+set @save_optimizer_switch= @@optimizer_switch;
+SET optimizer_switch='materialization=off';
+
+SELECT t1.a FROM t1
+WHERE t1.a IN (SELECT t2.a FROM t2 WHERE t2.b IN (SELECT t3.b FROM t3 WHERE t3.c= 27)) LIMIT 5;
+
+SET optimizer_switch='materialization=on';
+
+SELECT t1.a FROM t1
+WHERE t1.a IN (SELECT t2.a FROM t2 WHERE t2.b IN (SELECT t3.b FROM t3 WHERE t3.c= 27)) LIMIT 5;
+
+drop procedure prepare_data;
+set @@optimizer_switch= @save_optimizer_switch;
+drop table t1,t2,t3;
+
 --echo # End of 5.5 tests
diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc
index 783e3d4d22f..f472cf16710 100644
--- a/sql/opt_subselect.cc
+++ b/sql/opt_subselect.cc
@@ -440,6 +440,7 @@ bool subquery_types_allow_materialization(Item_in_subselect *in_subs);
 static bool replace_where_subcondition(JOIN *, Item **, Item *, Item *, bool);
 static int subq_sj_candidate_cmp(Item_in_subselect* el1, Item_in_subselect* el2,
                                  void *arg);
+static void reset_equality_number_for_subq_conds(Item * cond);
 static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred);
 static bool convert_subq_to_jtbm(JOIN *parent_join, 
                                  Item_in_subselect *subq_pred, bool *remove);
@@ -1456,6 +1457,67 @@ static int subq_sj_candidate_cmp(Item_in_subselect* el1, Item_in_subselect* el2,
 }
 
 
+/**
+    @brief
+    reset the value of the field in_eqaulity_no for all Item_func_eq
+    items in the where clause of the subquery.
+
+    Look for in_equality_no description in Item_func_eq class
+
+    DESCRIPTION
+    Lets have an example:
+    SELECT t1.a FROM t1 WHERE t1.a IN
+      (SELECT t2.a FROM t2 where t2.b IN
+          (select t3.b from t3 where t3.c=27 ))
+
+    So for such a query we have the parent, child and
+    grandchild select.
+
+    So for the equality t2.b = t3.b we set the value for in_equality_no to
+    0 according to its description. Wewe do the same for t1.a = t2.a.
+    But when we look at the child select (with the grandchild select merged),
+    the query would be
+
+    SELECT t1.a FROM t1 WHERE t1.a IN
+      (SELECT t2.a FROM t2 where t2.b = t3.b and t3.c=27)
+
+    and then when the child select is merged into the parent select the query
+    would look like
+
+    SELECT t1.a FROM t1, semi-join-nest(t2,t3)
+            WHERE t1.a =t2.a and t2.b = t3.b and t3.c=27
+
+    Still we would have in_equality_no set for t2.b = t3.b
+    though it does not take part in the semi-join equality for the parent select,
+    so we should reset its value to UINT_MAX.
+
+    @param cond WHERE clause of the subquery
+*/
+
+static void reset_equality_number_for_subq_conds(Item * cond)
+{
+  if (!cond)
+    return;
+  if (cond->type() == Item::COND_ITEM)
+  {
+    List_iterator<Item> li(*((Item_cond*) cond)->argument_list());
+    Item *item;
+    while ((item=li++))
+    {
+      if (item->type() == Item::FUNC_ITEM &&
+      ((Item_func*)item)->functype()== Item_func::EQ_FUNC)
+        ((Item_func_eq*)item)->in_equality_no= UINT_MAX;
+    }
+  }
+  else
+  {
+    if (cond->type() == Item::FUNC_ITEM &&
+      ((Item_func*)cond)->functype()== Item_func::EQ_FUNC)
+        ((Item_func_eq*)cond)->in_equality_no= UINT_MAX;
+  }
+  return;
+}
+
 /*
   Convert a subquery predicate into a TABLE_LIST semi-join nest
 
@@ -1713,6 +1775,7 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
   */
   sj_nest->sj_in_exprs= subq_pred->left_expr->cols();
   sj_nest->nested_join->sj_outer_expr_list.empty();
+  reset_equality_number_for_subq_conds(sj_nest->sj_on_expr);
 
   if (subq_pred->left_expr->cols() == 1)
   {

From aad70e9b4cdd719c3f7a559764e37ba7eea7bc3e Mon Sep 17 00:00:00 2001
From: Igor Babaev <igor@askmonty.org>
Date: Wed, 25 Jul 2018 11:57:57 -0700
Subject: [PATCH 062/164] MDEV-16820 Lost 'Impossible where' from query with
 inexpensive subquery

This patch fixes another problem introduced by the patch for mdev-4817.
The latter changed Item_cond::fix_fields() in such a way that it could
call the virtual method is_expensive(). With the first its call
the method saves the result in Item::is_expensive_cache. For all next
calls the method returns the result from this cache. So if the item
once was determined as expensive the method always returns true.
For subqueries it's not good, because non-optimized subqueries always
is considered as expensive.
It means that the cache should be invalidated after the call of
optimize_constant_subqueries().
---
 mysql-test/r/subselect.result                 | 16 ++++++++++++++++
 mysql-test/r/subselect_no_exists_to_in.result | 16 ++++++++++++++++
 mysql-test/r/subselect_no_mat.result          | 16 ++++++++++++++++
 mysql-test/r/subselect_no_opts.result         | 16 ++++++++++++++++
 mysql-test/r/subselect_no_scache.result       | 16 ++++++++++++++++
 mysql-test/r/subselect_no_semijoin.result     | 16 ++++++++++++++++
 mysql-test/t/subselect.test                   | 15 +++++++++++++++
 sql/item.h                                    |  5 +++++
 sql/sql_select.cc                             |  7 +++++++
 9 files changed, 123 insertions(+)

diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result
index e2ed43158f1..1c75aa66287 100644
--- a/mysql-test/r/subselect.result
+++ b/mysql-test/r/subselect.result
@@ -7240,6 +7240,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/subselect_no_exists_to_in.result b/mysql-test/r/subselect_no_exists_to_in.result
index 6306cfff89d..0bd14a535e0 100644
--- a/mysql-test/r/subselect_no_exists_to_in.result
+++ b/mysql-test/r/subselect_no_exists_to_in.result
@@ -7240,6 +7240,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/subselect_no_mat.result b/mysql-test/r/subselect_no_mat.result
index 0db1b031133..a3b7871bb7e 100644
--- a/mysql-test/r/subselect_no_mat.result
+++ b/mysql-test/r/subselect_no_mat.result
@@ -7233,6 +7233,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/subselect_no_opts.result b/mysql-test/r/subselect_no_opts.result
index 1fa6f385b2b..9dcc690448f 100644
--- a/mysql-test/r/subselect_no_opts.result
+++ b/mysql-test/r/subselect_no_opts.result
@@ -7231,6 +7231,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/subselect_no_scache.result b/mysql-test/r/subselect_no_scache.result
index 986542bd0bf..7b3c001f6b3 100644
--- a/mysql-test/r/subselect_no_scache.result
+++ b/mysql-test/r/subselect_no_scache.result
@@ -7246,6 +7246,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 # End of 10.0 tests
 #
diff --git a/mysql-test/r/subselect_no_semijoin.result b/mysql-test/r/subselect_no_semijoin.result
index d5fcc17f08c..a0625246e2b 100644
--- a/mysql-test/r/subselect_no_semijoin.result
+++ b/mysql-test/r/subselect_no_semijoin.result
@@ -7231,6 +7231,22 @@ a
 5
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16820: impossible where with inexpensive subquery
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+explain select * from t1 where (select max(b) from t2) = 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
+2	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
+drop table t1,t2;
 End of 5.5 tests
 # End of 10.0 tests
 #
diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test
index 7e9e5a731bc..1048e5a5a27 100644
--- a/mysql-test/t/subselect.test
+++ b/mysql-test/t/subselect.test
@@ -6102,6 +6102,21 @@ and t1.a in (select `test`.`t3`.`c` from `test`.`t3`);
 SET @@optimizer_switch= @optimiser_switch_save;
 DROP TABLE t1, t2, t3;
 
+--echo #
+--echo # MDEV-16820: impossible where with inexpensive subquery
+--echo #
+
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (1), (7);
+
+create table t2 (b int, index idx(b));
+insert into t2 values (2), (5), (3), (2);
+
+explain select * from t1 where (select max(b) from t2) = 10;
+explain select * from t1 where (select max(b) from t2) = 10 and t1.a > 3;
+
+drop table t1,t2;
+
 --echo End of 5.5 tests
 --echo # End of 10.0 tests
 
diff --git a/sql/item.h b/sql/item.h
index ce07f993dc4..8fad8dadf22 100644
--- a/sql/item.h
+++ b/sql/item.h
@@ -1614,6 +1614,11 @@ public:
   virtual bool limit_index_condition_pushdown_processor(void *arg) { return 0; }
   virtual bool exists2in_processor(void *arg) { return 0; }
   virtual bool find_selective_predicates_list_processor(void *arg) { return 0; }
+  bool cleanup_is_expensive_cache_processor(void *arg)
+  {
+    is_expensive_cache= (int8)(-1);
+    return 0;
+  }
 
   /* 
     TRUE if the expression depends only on the table indicated by tab_map
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 1fa21437266..4cabf5c1b40 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -1335,6 +1335,13 @@ JOIN::optimize_inner()
   if (optimize_constant_subqueries())
     DBUG_RETURN(1);
 
+  if (conds && conds->has_subquery())
+    (void) conds->walk(&Item::cleanup_is_expensive_cache_processor,
+                       0, (void *) 0);
+  if (having && having->has_subquery())
+    (void) having->walk(&Item::cleanup_is_expensive_cache_processor,
+			0, (void *) 0);
+
   if (setup_jtbm_semi_joins(this, join_list, &conds))
     DBUG_RETURN(1);
 

From 7f548943298f520d2cba635c4aa9af5262f4a0c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Tue, 24 Jul 2018 09:35:47 +0300
Subject: [PATCH 063/164] Remove mtr_set_flush_observer()

---
 storage/innobase/btr/btr0bulk.cc   | 22 +++++++++-------------
 storage/innobase/include/mtr0mtr.h |  7 -------
 2 files changed, 9 insertions(+), 20 deletions(-)

diff --git a/storage/innobase/btr/btr0bulk.cc b/storage/innobase/btr/btr0bulk.cc
index 5fae57fe2d4..39423fe318d 100644
--- a/storage/innobase/btr/btr0bulk.cc
+++ b/storage/innobase/btr/btr0bulk.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2014, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation.
+Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -50,10 +50,10 @@ PageBulk::init()
 
 	mtr = static_cast<mtr_t*>(
 		mem_heap_alloc(m_heap, sizeof(mtr_t)));
-	mtr_start(mtr);
+	mtr->start();
 	mtr_x_lock(dict_index_get_lock(m_index), mtr);
-	mtr_set_log_mode(mtr, MTR_LOG_NO_REDO);
-	mtr_set_flush_observer(mtr, m_flush_observer);
+	mtr->set_log_mode(MTR_LOG_NO_REDO);
+	mtr->set_flush_observer(m_flush_observer);
 
 	if (m_page_no == FIL_NULL) {
 		mtr_t	alloc_mtr;
@@ -597,18 +597,14 @@ PageBulk::release()
 dberr_t
 PageBulk::latch()
 {
-	ibool	ret;
-
-	mtr_start(m_mtr);
+	m_mtr->start();
 	mtr_x_lock(dict_index_get_lock(m_index), m_mtr);
-	mtr_set_log_mode(m_mtr, MTR_LOG_NO_REDO);
-	mtr_set_flush_observer(m_mtr, m_flush_observer);
+	m_mtr->set_log_mode(MTR_LOG_NO_REDO);
+	m_mtr->set_flush_observer(m_flush_observer);
 
-	/* TODO: need a simple and wait version of buf_page_optimistic_get. */
-	ret = buf_page_optimistic_get(RW_X_LATCH, m_block, m_modify_clock,
-				      __FILE__, __LINE__, m_mtr);
 	/* In case the block is S-latched by page_cleaner. */
-	if (!ret) {
+	if (!buf_page_optimistic_get(RW_X_LATCH, m_block, m_modify_clock,
+				     __FILE__, __LINE__, m_mtr)) {
 		page_id_t       page_id(dict_index_get_space(m_index), m_page_no);
 		page_size_t     page_size(dict_table_page_size(m_index->table));
 
diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h
index 6639a3448ea..bdd3a6a67b9 100644
--- a/storage/innobase/include/mtr0mtr.h
+++ b/storage/innobase/include/mtr0mtr.h
@@ -63,13 +63,6 @@ savepoint. */
 @return	old mode */
 #define mtr_set_log_mode(m, d)	(m)->set_log_mode((d))
 
-/** Get the flush observer of a mini-transaction.
-@return flush observer object */
-#define mtr_get_flush_observer(m)	(m)->get_flush_observer()
-
-/** Set the flush observer of a mini-transaction. */
-#define mtr_set_flush_observer(m, d)	(m)->set_flush_observer((d))
-
 /** Read 1 - 4 bytes from a file page buffered in the buffer pool.
 @return	value read */
 #define mtr_read_ulint(p, t, m)	(m)->read_ulint((p), (t))

From 172199b88cc9bc3f0765aef02a3a18ced8539d91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Tue, 24 Jul 2018 10:23:17 +0300
Subject: [PATCH 064/164] PageBulk: Remove dead code

Native ALTER TABLE is never invoked on temporary tables.
---
 storage/innobase/btr/btr0bulk.cc    | 8 ++------
 storage/innobase/include/btr0bulk.h | 1 +
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/storage/innobase/btr/btr0bulk.cc b/storage/innobase/btr/btr0bulk.cc
index 39423fe318d..86b233a8884 100644
--- a/storage/innobase/btr/btr0bulk.cc
+++ b/storage/innobase/btr/btr0bulk.cc
@@ -122,9 +122,7 @@ PageBulk::init()
 		btr_page_set_level(new_page, NULL, m_level, mtr);
 	}
 
-	if (dict_index_is_sec_or_ibuf(m_index)
-	    && !dict_table_is_temporary(m_index->table)
-	    && page_is_leaf(new_page)) {
+	if (!m_level && dict_index_is_sec_or_ibuf(m_index)) {
 		page_update_max_trx_id(new_block, NULL, m_trx_id, mtr);
 	}
 
@@ -310,9 +308,7 @@ PageBulk::commit(
 		ut_ad(page_validate(m_page, m_index));
 
 		/* Set no free space left and no buffered changes in ibuf. */
-		if (!dict_index_is_clust(m_index)
-		    && !dict_table_is_temporary(m_index->table)
-		    && page_is_leaf(m_page)) {
+		if (!dict_index_is_clust(m_index) && page_is_leaf(m_page)) {
 			ibuf_set_bitmap_for_bulk_load(
 				m_block, innobase_fill_factor == 100);
 		}
diff --git a/storage/innobase/include/btr0bulk.h b/storage/innobase/include/btr0bulk.h
index a1887c3df2b..8c82b3c9708 100644
--- a/storage/innobase/include/btr0bulk.h
+++ b/storage/innobase/include/btr0bulk.h
@@ -84,6 +84,7 @@ public:
 		m_err(DB_SUCCESS)
 	{
 		ut_ad(!dict_index_is_spatial(m_index));
+		ut_ad(!dict_table_is_temporary(m_index->table));
 	}
 
 	/** Deconstructor */

From a3b22147ca7ed31625e500a1e0731769724e7ed7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Tue, 24 Jul 2018 10:46:30 +0300
Subject: [PATCH 065/164] Remove pointer indirection for PageBulk::m_mtr

---
 storage/innobase/btr/btr0bulk.cc    | 62 ++++++++++++++---------------
 storage/innobase/include/btr0bulk.h |  6 +--
 2 files changed, 32 insertions(+), 36 deletions(-)

diff --git a/storage/innobase/btr/btr0bulk.cc b/storage/innobase/btr/btr0bulk.cc
index 86b233a8884..e528156be92 100644
--- a/storage/innobase/btr/btr0bulk.cc
+++ b/storage/innobase/btr/btr0bulk.cc
@@ -39,7 +39,6 @@ Note: we commit all mtrs on failure.
 dberr_t
 PageBulk::init()
 {
-	mtr_t*		mtr;
 	buf_block_t*	new_block;
 	page_t*		new_page;
 	page_zip_des_t*	new_page_zip;
@@ -48,12 +47,10 @@ PageBulk::init()
 	ut_ad(m_heap == NULL);
 	m_heap = mem_heap_create(1000);
 
-	mtr = static_cast<mtr_t*>(
-		mem_heap_alloc(m_heap, sizeof(mtr_t)));
-	mtr->start();
-	mtr_x_lock(dict_index_get_lock(m_index), mtr);
-	mtr->set_log_mode(MTR_LOG_NO_REDO);
-	mtr->set_flush_observer(m_flush_observer);
+	m_mtr.start();
+	mtr_x_lock(&m_index->lock, &m_mtr);
+	m_mtr.set_log_mode(MTR_LOG_NO_REDO);
+	m_mtr.set_flush_observer(m_flush_observer);
 
 	if (m_page_no == FIL_NULL) {
 		mtr_t	alloc_mtr;
@@ -70,21 +67,21 @@ PageBulk::init()
 		success = fsp_reserve_free_extents(&n_reserved, m_index->space,
 						   1, FSP_NORMAL, &alloc_mtr);
 		if (!success) {
-			mtr_commit(&alloc_mtr);
-			mtr_commit(mtr);
+			alloc_mtr.commit();
+			m_mtr.commit();
 			return(DB_OUT_OF_FILE_SPACE);
 		}
 
 		/* Allocate a new page. */
 		new_block = btr_page_alloc(m_index, 0, FSP_UP, m_level,
-					   &alloc_mtr, mtr);
+					   &alloc_mtr, &m_mtr);
 
 		if (n_reserved > 0) {
 			fil_space_release_free_extents(m_index->space,
 						       n_reserved);
 		}
 
-		mtr_commit(&alloc_mtr);
+		alloc_mtr.commit();
 
 		new_page = buf_block_get_frame(new_block);
 		new_page_zip = buf_block_get_page_zip(new_block);
@@ -92,25 +89,25 @@ PageBulk::init()
 
 		if (new_page_zip) {
 			page_create_zip(new_block, m_index, m_level, 0,
-					NULL, mtr);
+					NULL, &m_mtr);
 		} else {
 			ut_ad(!dict_index_is_spatial(m_index));
-			page_create(new_block, mtr,
+			page_create(new_block, &m_mtr,
 				    dict_table_is_comp(m_index->table),
 				    false);
-			btr_page_set_level(new_page, NULL, m_level, mtr);
+			btr_page_set_level(new_page, NULL, m_level, &m_mtr);
 		}
 
-		btr_page_set_next(new_page, NULL, FIL_NULL, mtr);
-		btr_page_set_prev(new_page, NULL, FIL_NULL, mtr);
+		btr_page_set_next(new_page, NULL, FIL_NULL, &m_mtr);
+		btr_page_set_prev(new_page, NULL, FIL_NULL, &m_mtr);
 
-		btr_page_set_index_id(new_page, NULL, m_index->id, mtr);
+		btr_page_set_index_id(new_page, NULL, m_index->id, &m_mtr);
 	} else {
 		page_id_t	page_id(dict_index_get_space(m_index), m_page_no);
 		page_size_t	page_size(dict_table_page_size(m_index->table));
 
 		new_block = btr_block_get(page_id, page_size,
-					  RW_X_LATCH, m_index, mtr);
+					  RW_X_LATCH, m_index, &m_mtr);
 
 		new_page = buf_block_get_frame(new_block);
 		new_page_zip = buf_block_get_page_zip(new_block);
@@ -119,14 +116,13 @@ PageBulk::init()
 
 		ut_ad(page_dir_get_n_heap(new_page) == PAGE_HEAP_NO_USER_LOW);
 
-		btr_page_set_level(new_page, NULL, m_level, mtr);
+		btr_page_set_level(new_page, NULL, m_level, &m_mtr);
 	}
 
 	if (!m_level && dict_index_is_sec_or_ibuf(m_index)) {
-		page_update_max_trx_id(new_block, NULL, m_trx_id, mtr);
+		page_update_max_trx_id(new_block, NULL, m_trx_id, &m_mtr);
 	}
 
-	m_mtr = mtr;
 	m_block = new_block;
 	m_block->skip_flush_check = true;
 	m_page = new_page;
@@ -314,7 +310,7 @@ PageBulk::commit(
 		}
 	}
 
-	mtr_commit(m_mtr);
+	m_mtr.commit();
 }
 
 /** Compress a page of compressed table
@@ -326,7 +322,7 @@ PageBulk::compress()
 	ut_ad(m_page_zip != NULL);
 
 	return(page_zip_compress(m_page_zip, m_page, m_index,
-				 page_zip_level, NULL, m_mtr));
+				 page_zip_level, NULL, &m_mtr));
 }
 
 /** Get node pointer
@@ -478,7 +474,7 @@ void
 PageBulk::setNext(
 	ulint		next_page_no)
 {
-	btr_page_set_next(m_page, NULL, next_page_no, m_mtr);
+	btr_page_set_next(m_page, NULL, next_page_no, &m_mtr);
 }
 
 /** Set previous page
@@ -487,7 +483,7 @@ void
 PageBulk::setPrev(
 	ulint		prev_page_no)
 {
-	btr_page_set_prev(m_page, NULL, prev_page_no, m_mtr);
+	btr_page_set_prev(m_page, NULL, prev_page_no, &m_mtr);
 }
 
 /** Check if required space is available in the page for the rec to be inserted.
@@ -560,7 +556,7 @@ PageBulk::storeExt(
 	page_cur->block = m_block;
 
 	dberr_t	err = btr_store_big_rec_extern_fields(
-		&btr_pcur, offsets, big_rec, m_mtr, BTR_STORE_INSERT_BULK);
+		&btr_pcur, offsets, big_rec, &m_mtr, BTR_STORE_INSERT_BULK);
 
 	ut_ad(page_offset(m_cur_rec) == page_offset(page_cur->rec));
 
@@ -586,27 +582,27 @@ PageBulk::release()
 	/* No other threads can modify this block. */
 	m_modify_clock = buf_block_get_modify_clock(m_block);
 
-	mtr_commit(m_mtr);
+	m_mtr.commit();
 }
 
 /** Start mtr and latch the block */
 dberr_t
 PageBulk::latch()
 {
-	m_mtr->start();
-	mtr_x_lock(dict_index_get_lock(m_index), m_mtr);
-	m_mtr->set_log_mode(MTR_LOG_NO_REDO);
-	m_mtr->set_flush_observer(m_flush_observer);
+	m_mtr.start();
+	mtr_x_lock(&m_index->lock, &m_mtr);
+	m_mtr.set_log_mode(MTR_LOG_NO_REDO);
+	m_mtr.set_flush_observer(m_flush_observer);
 
 	/* In case the block is S-latched by page_cleaner. */
 	if (!buf_page_optimistic_get(RW_X_LATCH, m_block, m_modify_clock,
-				     __FILE__, __LINE__, m_mtr)) {
+				     __FILE__, __LINE__, &m_mtr)) {
 		page_id_t       page_id(dict_index_get_space(m_index), m_page_no);
 		page_size_t     page_size(dict_table_page_size(m_index->table));
 
 		m_block = buf_page_get_gen(page_id, page_size, RW_X_LATCH,
 					   m_block, BUF_GET_IF_IN_POOL,
-					   __FILE__, __LINE__, m_mtr, &m_err);
+					   __FILE__, __LINE__, &m_mtr, &m_err);
 
 		if (m_err != DB_SUCCESS) {
 			return (m_err);
diff --git a/storage/innobase/include/btr0bulk.h b/storage/innobase/include/btr0bulk.h
index 8c82b3c9708..e6ef2fa4a03 100644
--- a/storage/innobase/include/btr0bulk.h
+++ b/storage/innobase/include/btr0bulk.h
@@ -63,7 +63,7 @@ public:
 		:
 		m_heap(NULL),
 		m_index(index),
-		m_mtr(NULL),
+		m_mtr(),
 		m_trx_id(trx_id),
 		m_block(NULL),
 		m_page(NULL),
@@ -206,8 +206,8 @@ private:
 	/** The index B-tree */
 	dict_index_t*	m_index;
 
-	/** The min-transaction */
-	mtr_t*		m_mtr;
+	/** The mini-transaction */
+	mtr_t		m_mtr;
 
 	/** The transaction id */
 	trx_id_t	m_trx_id;

From 4a456eac2a27817731697d00057a9714f0e20caa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Tue, 24 Jul 2018 10:48:31 +0300
Subject: [PATCH 066/164] Remove unused BtrBulk::m_heap

---
 storage/innobase/btr/btr0bulk.cc    | 2 --
 storage/innobase/include/btr0bulk.h | 8 --------
 2 files changed, 10 deletions(-)

diff --git a/storage/innobase/btr/btr0bulk.cc b/storage/innobase/btr/btr0bulk.cc
index e528156be92..ec2fea64cc3 100644
--- a/storage/innobase/btr/btr0bulk.cc
+++ b/storage/innobase/btr/btr0bulk.cc
@@ -763,8 +763,6 @@ BtrBulk::insert(
 	bool		is_left_most = false;
 	dberr_t		err = DB_SUCCESS;
 
-	ut_ad(m_heap != NULL);
-
 	/* Check if we need to create a PageBulk for the level. */
 	if (level + 1 > m_page_bulks->size()) {
 		PageBulk*	new_page_bulk
diff --git a/storage/innobase/include/btr0bulk.h b/storage/innobase/include/btr0bulk.h
index e6ef2fa4a03..9048833c7a9 100644
--- a/storage/innobase/include/btr0bulk.h
+++ b/storage/innobase/include/btr0bulk.h
@@ -279,7 +279,6 @@ public:
 		trx_id_t	trx_id,
 		FlushObserver*	observer)
 		:
-		m_heap(NULL),
 		m_index(index),
 		m_trx_id(trx_id),
 		m_flush_observer(observer)
@@ -293,7 +292,6 @@ public:
 	/** Destructor */
 	~BtrBulk()
 	{
-		mem_heap_free(m_heap);
 		UT_DELETE(m_page_bulks);
 
 #ifdef UNIV_DEBUG
@@ -305,9 +303,6 @@ public:
 	Note: must be called right after constructor. */
 	void init()
 	{
-		ut_ad(m_heap == NULL);
-		m_heap = mem_heap_create(1000);
-
 		m_page_bulks = UT_NEW_NOKEY(page_bulk_vector());
 	}
 
@@ -371,9 +366,6 @@ private:
 	void logFreeCheck();
 
 private:
-	/** Memory heap for allocation */
-	mem_heap_t*		m_heap;
-
 	/** B-tree index */
 	dict_index_t*		m_index;
 

From 32eb5823e42d488c2c8eb85fdf822733e0a482a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Tue, 24 Jul 2018 10:54:15 +0300
Subject: [PATCH 067/164] Remove pointer indirection for BtrBulk::m_page_bulks

---
 storage/innobase/btr/btr0bulk.cc    | 32 ++++++++++++++---------------
 storage/innobase/include/btr0bulk.h | 11 +---------
 storage/innobase/row/row0ftsort.cc  |  1 -
 storage/innobase/row/row0merge.cc   |  4 ----
 4 files changed, 17 insertions(+), 31 deletions(-)

diff --git a/storage/innobase/btr/btr0bulk.cc b/storage/innobase/btr/btr0bulk.cc
index ec2fea64cc3..692ce90d8b8 100644
--- a/storage/innobase/btr/btr0bulk.cc
+++ b/storage/innobase/btr/btr0bulk.cc
@@ -730,10 +730,10 @@ BtrBulk::logFreeCheck()
 void
 BtrBulk::release()
 {
-	ut_ad(m_root_level + 1 == m_page_bulks->size());
+	ut_ad(m_root_level + 1 == m_page_bulks.size());
 
 	for (ulint level = 0; level <= m_root_level; level++) {
-		PageBulk*    page_bulk = m_page_bulks->at(level);
+		PageBulk*    page_bulk = m_page_bulks.at(level);
 
 		page_bulk->release();
 	}
@@ -743,10 +743,10 @@ BtrBulk::release()
 void
 BtrBulk::latch()
 {
-	ut_ad(m_root_level + 1 == m_page_bulks->size());
+	ut_ad(m_root_level + 1 == m_page_bulks.size());
 
 	for (ulint level = 0; level <= m_root_level; level++) {
-		PageBulk*    page_bulk = m_page_bulks->at(level);
+		PageBulk*    page_bulk = m_page_bulks.at(level);
 		page_bulk->latch();
 	}
 }
@@ -764,7 +764,7 @@ BtrBulk::insert(
 	dberr_t		err = DB_SUCCESS;
 
 	/* Check if we need to create a PageBulk for the level. */
-	if (level + 1 > m_page_bulks->size()) {
+	if (level + 1 > m_page_bulks.size()) {
 		PageBulk*	new_page_bulk
 			= UT_NEW_NOKEY(PageBulk(m_index, m_trx_id, FIL_NULL,
 						level, m_flush_observer));
@@ -773,16 +773,16 @@ BtrBulk::insert(
 			return(err);
 		}
 
-		m_page_bulks->push_back(new_page_bulk);
-		ut_ad(level + 1 == m_page_bulks->size());
+		m_page_bulks.push_back(new_page_bulk);
+		ut_ad(level + 1 == m_page_bulks.size());
 		m_root_level = level;
 
 		is_left_most = true;
 	}
 
-	ut_ad(m_page_bulks->size() > level);
+	ut_ad(m_page_bulks.size() > level);
 
-	PageBulk*	page_bulk = m_page_bulks->at(level);
+	PageBulk*	page_bulk = m_page_bulks.at(level);
 
 	if (is_left_most && level > 0 && page_bulk->getRecNo() == 0) {
 		/* The node pointer must be marked as the predefined minimum
@@ -838,7 +838,7 @@ BtrBulk::insert(
 
 		/* Set new page bulk to page_bulks. */
 		ut_ad(sibling_page_bulk->getLevel() <= m_root_level);
-		m_page_bulks->at(level) = sibling_page_bulk;
+		m_page_bulks.at(level) = sibling_page_bulk;
 
 		UT_DELETE(page_bulk);
 		page_bulk = sibling_page_bulk;
@@ -871,11 +871,11 @@ BtrBulk::insert(
 	if (big_rec != NULL) {
 		ut_ad(dict_index_is_clust(m_index));
 		ut_ad(page_bulk->getLevel() == 0);
-		ut_ad(page_bulk == m_page_bulks->at(0));
+		ut_ad(page_bulk == m_page_bulks.at(0));
 
 		/* Release all latched but leaf node. */
 		for (ulint level = 1; level <= m_root_level; level++) {
-			PageBulk*    page_bulk = m_page_bulks->at(level);
+			PageBulk*    page_bulk = m_page_bulks.at(level);
 
 			page_bulk->release();
 		}
@@ -884,7 +884,7 @@ BtrBulk::insert(
 
 		/* Latch */
 		for (ulint level = 1; level <= m_root_level; level++) {
-			PageBulk*    page_bulk = m_page_bulks->at(level);
+			PageBulk*    page_bulk = m_page_bulks.at(level);
 			page_bulk->latch();
 		}
 	}
@@ -909,17 +909,17 @@ BtrBulk::finish(dberr_t	err)
 
 	ut_ad(!dict_table_is_temporary(m_index->table));
 
-	if (m_page_bulks->size() == 0) {
+	if (m_page_bulks.size() == 0) {
 		/* The table is empty. The root page of the index tree
 		is already in a consistent state. No need to flush. */
 		return(err);
 	}
 
-	ut_ad(m_root_level + 1 == m_page_bulks->size());
+	ut_ad(m_root_level + 1 == m_page_bulks.size());
 
 	/* Finish all page bulks */
 	for (ulint level = 0; level <= m_root_level; level++) {
-		PageBulk*	page_bulk = m_page_bulks->at(level);
+		PageBulk*	page_bulk = m_page_bulks.at(level);
 
 		last_page_no = page_bulk->getPageNo();
 
diff --git a/storage/innobase/include/btr0bulk.h b/storage/innobase/include/btr0bulk.h
index 9048833c7a9..a3af0ea7c93 100644
--- a/storage/innobase/include/btr0bulk.h
+++ b/storage/innobase/include/btr0bulk.h
@@ -292,20 +292,11 @@ public:
 	/** Destructor */
 	~BtrBulk()
 	{
-		UT_DELETE(m_page_bulks);
-
 #ifdef UNIV_DEBUG
 		fil_space_dec_redo_skipped_count(m_index->space);
 #endif /* UNIV_DEBUG */
 	}
 
-	/** Initialization
-	Note: must be called right after constructor. */
-	void init()
-	{
-		m_page_bulks = UT_NEW_NOKEY(page_bulk_vector());
-	}
-
 	/** Insert a tuple
 	@param[in]	tuple	tuple to insert.
 	@return error code */
@@ -379,7 +370,7 @@ private:
 	FlushObserver*		m_flush_observer;
 
 	/** Page cursor vector for all level */
-	page_bulk_vector*	m_page_bulks;
+	page_bulk_vector	m_page_bulks;
 };
 
 #endif
diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc
index 0a937847c33..c59d3a95e3e 100644
--- a/storage/innobase/row/row0ftsort.cc
+++ b/storage/innobase/row/row0ftsort.cc
@@ -1689,7 +1689,6 @@ row_fts_merge_insert(
 
 	/* Create bulk load instance */
 	ins_ctx.btr_bulk = UT_NEW_NOKEY(BtrBulk(aux_index, trx->id, observer));
-	ins_ctx.btr_bulk->init();
 
 	/* Create tuple for insert */
 	ins_ctx.tuple = dtuple_create(heap, dict_index_get_n_fields(aux_index));
diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc
index 66f3b3936af..8edea4959f1 100644
--- a/storage/innobase/row/row0merge.cc
+++ b/storage/innobase/row/row0merge.cc
@@ -2404,8 +2404,6 @@ write_buffers:
 							BtrBulk(index[i],
 								trx->id,
 								observer));
-
-						clust_btr_bulk->init();
 					} else {
 						clust_btr_bulk->latch();
 					}
@@ -2519,7 +2517,6 @@ write_buffers:
 
 					BtrBulk	btr_bulk(index[i], trx->id,
 							 observer);
-					btr_bulk.init();
 
 					err = row_merge_insert_index_tuples(
 						trx->id, index[i], old_table,
@@ -4888,7 +4885,6 @@ wait_again:
 			if (error == DB_SUCCESS) {
 				BtrBulk	btr_bulk(sort_idx, trx->id,
 						 flush_observer);
-				btr_bulk.init();
 
 				pct_cost = (COST_BUILD_INDEX_STATIC +
 					(total_dynamic_cost * merge_files[i].offset /

From 0f90728bc0f8bc946a61500801b23f8a316e73d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Wed, 25 Jul 2018 17:05:47 +0300
Subject: [PATCH 068/164] MDEV-16809 Allow full redo logging for ALTER TABLE

Introduce the configuration option innodb_log_optimize_ddl
for controlling whether native index creation or table-rebuild
in InnoDB should keep optimizing the redo log
(and writing MLOG_INDEX_LOAD records to ensure that
concurrent backup would fail).

By default, we have innodb_log_optimize_ddl=ON, that is,
the default behaviour that was introduced in MariaDB 10.2.2
(with the merge of InnoDB from MySQL 5.7) will be unchanged.

BtrBulk::m_trx: Replaces m_trx_id. We must be able to check for
KILL QUERY even if !m_flush_observer (innodb_log_optimize_ddl=OFF).

page_cur_insert_rec_write_log(): Declare globally, so that this
can be called from PageBulk::insert().

row_merge_insert_index_tuples(): Remove the unused parameter trx_id.

row_merge_build_indexes(): Enable or disable redo logging based on
the innodb_log_optimize_ddl parameter.

PageBulk::init(), PageBulk::insert(), PageBulk::finish(): Write
redo log records if needed. For ROW_FORMAT=COMPRESSED, redo log
will be written in PageBulk::compress() unless we called
m_mtr.set_log_mode(MTR_LOG_NO_REDO).
---
 .../innodb_log_optimize_ddl.result            |  37 ++++
 .../mariabackup/innodb_log_optimize_ddl.test  |  47 ++++++
 .../suite/sys_vars/r/sysvars_innodb.result    |  14 ++
 storage/innobase/btr/btr0bulk.cc              | 159 +++++++++++++-----
 storage/innobase/buf/buf0flu.cc               |  10 +-
 storage/innobase/handler/ha_innodb.cc         |   8 +
 storage/innobase/include/btr0bulk.h           |  29 ++--
 storage/innobase/include/buf0flu.h            |   7 +-
 storage/innobase/include/page0cur.h           |  15 ++
 storage/innobase/page/page0cur.cc             |  22 +--
 storage/innobase/row/row0ftsort.cc            |   7 +-
 storage/innobase/row/row0merge.cc             |  39 ++---
 12 files changed, 291 insertions(+), 103 deletions(-)
 create mode 100644 mysql-test/suite/mariabackup/innodb_log_optimize_ddl.result
 create mode 100644 mysql-test/suite/mariabackup/innodb_log_optimize_ddl.test

diff --git a/mysql-test/suite/mariabackup/innodb_log_optimize_ddl.result b/mysql-test/suite/mariabackup/innodb_log_optimize_ddl.result
new file mode 100644
index 00000000000..47b74e0f709
--- /dev/null
+++ b/mysql-test/suite/mariabackup/innodb_log_optimize_ddl.result
@@ -0,0 +1,37 @@
+SET GLOBAL innodb_log_optimize_ddl=OFF;
+CREATE TABLE tz(id BIGINT PRIMARY KEY, i INT)
+ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
+INSERT INTO tz(id) select * from seq_1_to_10000;
+CREATE TABLE tr(id BIGINT PRIMARY KEY, i INT)
+ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+INSERT INTO tr(id) select * from seq_1_to_10000;
+CREATE TABLE td(id BIGINT PRIMARY KEY, i INT)
+ENGINE=InnoDB;
+INSERT INTO td(id) select * from seq_1_to_10000;
+CREATE PROCEDURE a()
+BEGIN
+ALTER TABLE tz ADD INDEX(i);
+ALTER TABLE tr ADD INDEX(i);
+ALTER TABLE td ADD INDEX(i);
+END //
+call a();
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+DROP PROCEDURE a;
+CHECK TABLE tz,tr,td;
+Table	Op	Msg_type	Msg_text
+test.tz	check	status	OK
+test.tr	check	status	OK
+test.td	check	status	OK
+SELECT COUNT(*) FROM tz;
+COUNT(*)
+10000
+SELECT COUNT(*) FROM tr;
+COUNT(*)
+10000
+SELECT COUNT(*) FROM td;
+COUNT(*)
+10000
+DROP TABLE tz,tr,td;
diff --git a/mysql-test/suite/mariabackup/innodb_log_optimize_ddl.test b/mysql-test/suite/mariabackup/innodb_log_optimize_ddl.test
new file mode 100644
index 00000000000..d6680bec077
--- /dev/null
+++ b/mysql-test/suite/mariabackup/innodb_log_optimize_ddl.test
@@ -0,0 +1,47 @@
+# see unsupported_redo.test for the opposite (default) case
+--source include/have_innodb.inc
+--source include/have_sequence.inc
+
+SET GLOBAL innodb_log_optimize_ddl=OFF;
+
+CREATE TABLE tz(id BIGINT PRIMARY KEY, i INT)
+ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
+INSERT INTO tz(id) select * from seq_1_to_10000;
+CREATE TABLE tr(id BIGINT PRIMARY KEY, i INT)
+ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+INSERT INTO tr(id) select * from seq_1_to_10000;
+CREATE TABLE td(id BIGINT PRIMARY KEY, i INT)
+ENGINE=InnoDB;
+INSERT INTO td(id) select * from seq_1_to_10000;
+
+DELIMITER //;
+CREATE PROCEDURE a()
+BEGIN
+ ALTER TABLE tz ADD INDEX(i);
+ ALTER TABLE tr ADD INDEX(i);
+ ALTER TABLE td ADD INDEX(i);
+END //
+DELIMITER ;//
+
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+
+send call a();
+
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir;
+--enable_result_log
+exec $XTRABACKUP  --prepare --target-dir=$targetdir;
+
+reap;
+
+-- source include/restart_and_restore.inc
+--rmdir $targetdir
+
+DROP PROCEDURE a;
+
+CHECK TABLE tz,tr,td;
+SELECT COUNT(*) FROM tz;
+SELECT COUNT(*) FROM tr;
+SELECT COUNT(*) FROM td;
+
+DROP TABLE tz,tr,td;
diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result
index 894f9a95ccb..9d02435d815 100644
--- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result
+++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result
@@ -1798,6 +1798,20 @@ NUMERIC_BLOCK_SIZE	NULL
 ENUM_VALUE_LIST	NULL
 READ_ONLY	YES
 COMMAND_LINE_ARGUMENT	REQUIRED
+VARIABLE_NAME	INNODB_LOG_OPTIMIZE_DDL
+SESSION_VALUE	NULL
+GLOBAL_VALUE	ON
+GLOBAL_VALUE_ORIGIN	COMPILE-TIME
+DEFAULT_VALUE	ON
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	BOOLEAN
+VARIABLE_COMMENT	Reduce redo logging when natively creating indexes or rebuilding tables. Setting this OFF avoids delay due to page flushing and allows concurrent backup.
+NUMERIC_MIN_VALUE	NULL
+NUMERIC_MAX_VALUE	NULL
+NUMERIC_BLOCK_SIZE	NULL
+ENUM_VALUE_LIST	OFF,ON
+READ_ONLY	NO
+COMMAND_LINE_ARGUMENT	OPTIONAL
 VARIABLE_NAME	INNODB_LOG_WRITE_AHEAD_SIZE
 SESSION_VALUE	NULL
 GLOBAL_VALUE	8192
diff --git a/storage/innobase/btr/btr0bulk.cc b/storage/innobase/btr/btr0bulk.cc
index 692ce90d8b8..bf33745f091 100644
--- a/storage/innobase/btr/btr0bulk.cc
+++ b/storage/innobase/btr/btr0bulk.cc
@@ -29,9 +29,12 @@ Created 03/11/2014 Shaohua Wang
 #include "btr0cur.h"
 #include "btr0pcur.h"
 #include "ibuf0ibuf.h"
+#include "trx0trx.h"
 
 /** Innodb B-tree index fill factor for bulk load. */
 long	innobase_fill_factor;
+/** whether to reduce redo logging during ALTER TABLE */
+my_bool	innodb_log_optimize_ddl;
 
 /** Initialize members, allocate page if needed and start mtr.
 Note: we commit all mtrs on failure.
@@ -49,8 +52,12 @@ PageBulk::init()
 
 	m_mtr.start();
 	mtr_x_lock(&m_index->lock, &m_mtr);
-	m_mtr.set_log_mode(MTR_LOG_NO_REDO);
-	m_mtr.set_flush_observer(m_flush_observer);
+	if (m_flush_observer) {
+		m_mtr.set_log_mode(MTR_LOG_NO_REDO);
+		m_mtr.set_flush_observer(m_flush_observer);
+	} else {
+		m_mtr.set_named_space(m_index->space);
+	}
 
 	if (m_page_no == FIL_NULL) {
 		mtr_t	alloc_mtr;
@@ -59,8 +66,8 @@ PageBulk::init()
 		because we don't guarantee pages are committed following
 		the allocation order, and we will always generate redo log
 		for page allocation, even when creating a new tablespace. */
-		mtr_start(&alloc_mtr);
-		alloc_mtr.set_named_space(dict_index_get_space(m_index));
+		alloc_mtr.start();
+		alloc_mtr.set_named_space(m_index->space);
 
 		ulint	n_reserved;
 		bool	success;
@@ -90,18 +97,29 @@ PageBulk::init()
 		if (new_page_zip) {
 			page_create_zip(new_block, m_index, m_level, 0,
 					NULL, &m_mtr);
+			memset(FIL_PAGE_PREV + new_page, 0xff, 8);
+			page_zip_write_header(new_page_zip,
+					      FIL_PAGE_PREV + new_page,
+					      8, &m_mtr);
+			mach_write_to_8(PAGE_HEADER + PAGE_INDEX_ID + new_page,
+					m_index->id);
+			page_zip_write_header(new_page_zip,
+					      PAGE_HEADER + PAGE_INDEX_ID
+					      + new_page, 8, &m_mtr);
 		} else {
 			ut_ad(!dict_index_is_spatial(m_index));
 			page_create(new_block, &m_mtr,
 				    dict_table_is_comp(m_index->table),
 				    false);
-			btr_page_set_level(new_page, NULL, m_level, &m_mtr);
+			mlog_write_ulint(FIL_PAGE_PREV + new_page, FIL_NULL,
+					 MLOG_4BYTES, &m_mtr);
+			mlog_write_ulint(FIL_PAGE_NEXT + new_page, FIL_NULL,
+					 MLOG_4BYTES, &m_mtr);
+			mlog_write_ulint(PAGE_HEADER + PAGE_LEVEL + new_page,
+					 m_level, MLOG_2BYTES, &m_mtr);
+			mlog_write_ull(PAGE_HEADER + PAGE_INDEX_ID + new_page,
+				       m_index->id, &m_mtr);
 		}
-
-		btr_page_set_next(new_page, NULL, FIL_NULL, &m_mtr);
-		btr_page_set_prev(new_page, NULL, FIL_NULL, &m_mtr);
-
-		btr_page_set_index_id(new_page, NULL, m_index->id, &m_mtr);
 	} else {
 		page_id_t	page_id(dict_index_get_space(m_index), m_page_no);
 		page_size_t	page_size(dict_table_page_size(m_index->table));
@@ -116,11 +134,12 @@ PageBulk::init()
 
 		ut_ad(page_dir_get_n_heap(new_page) == PAGE_HEAP_NO_USER_LOW);
 
-		btr_page_set_level(new_page, NULL, m_level, &m_mtr);
+		btr_page_set_level(new_page, new_page_zip, m_level, &m_mtr);
 	}
 
 	if (!m_level && dict_index_is_sec_or_ibuf(m_index)) {
-		page_update_max_trx_id(new_block, NULL, m_trx_id, &m_mtr);
+		page_update_max_trx_id(new_block, new_page_zip, m_trx_id,
+				       &m_mtr);
 	}
 
 	m_block = new_block;
@@ -146,7 +165,9 @@ PageBulk::init()
 	m_rec_no = page_header_get_field(new_page, PAGE_N_RECS);
 
 	ut_d(m_total_data = 0);
-	page_header_set_field(m_page, NULL, PAGE_HEAP_TOP, UNIV_PAGE_SIZE - 1);
+	/* See page_copy_rec_list_end_to_created_page() */
+	ut_d(page_header_set_field(m_page, NULL, PAGE_HEAP_TOP,
+				   srv_page_size - 1));
 
 	return(DB_SUCCESS);
 }
@@ -213,6 +234,14 @@ PageBulk::insert(
 	m_free_space -= rec_size + slot_size;
 	m_heap_top += rec_size;
 	m_rec_no += 1;
+
+	if (!m_flush_observer && !m_page_zip) {
+		/* For ROW_FORMAT=COMPRESSED, redo log may be written
+		in PageBulk::compress(). */
+		page_cur_insert_rec_write_log(insert_rec, rec_size,
+					      m_cur_rec, m_index, &m_mtr);
+	}
+
 	m_cur_rec = insert_rec;
 }
 
@@ -223,15 +252,10 @@ void
 PageBulk::finish()
 {
 	ut_ad(m_rec_no > 0);
-
-#ifdef UNIV_DEBUG
 	ut_ad(m_total_data + page_dir_calc_reserved_space(m_rec_no)
 	      <= page_get_free_space_of_empty(m_is_comp));
-
-	/* To pass the debug tests we have to set these dummy values
-	in the debug version */
-	page_dir_set_n_slots(m_page, NULL, UNIV_PAGE_SIZE / 2);
-#endif
+	/* See page_copy_rec_list_end_to_created_page() */
+	ut_d(page_dir_set_n_slots(m_page, NULL, srv_page_size / 2));
 
 	ulint	count = 0;
 	ulint	n_recs = 0;
@@ -282,14 +306,43 @@ PageBulk::finish()
 	page_dir_slot_set_n_owned(slot, NULL, count + 1);
 
 	ut_ad(!dict_index_is_spatial(m_index));
-	page_dir_set_n_slots(m_page, NULL, 2 + slot_index);
-	page_header_set_ptr(m_page, NULL, PAGE_HEAP_TOP, m_heap_top);
-	page_dir_set_n_heap(m_page, NULL, PAGE_HEAP_NO_USER_LOW + m_rec_no);
-	page_header_set_field(m_page, NULL, PAGE_N_RECS, m_rec_no);
 
-	page_header_set_ptr(m_page, NULL, PAGE_LAST_INSERT, m_cur_rec);
-	page_header_set_field(m_page, NULL, PAGE_DIRECTION, PAGE_RIGHT);
-	page_header_set_field(m_page, NULL, PAGE_N_DIRECTION, 0);
+	if (!m_flush_observer && !m_page_zip) {
+		mlog_write_ulint(PAGE_HEADER + PAGE_N_DIR_SLOTS + m_page,
+				 2 + slot_index, MLOG_2BYTES, &m_mtr);
+		mlog_write_ulint(PAGE_HEADER + PAGE_HEAP_TOP + m_page,
+				 ulint(m_heap_top - m_page),
+				 MLOG_2BYTES, &m_mtr);
+		mlog_write_ulint(PAGE_HEADER + PAGE_N_HEAP + m_page,
+				 (PAGE_HEAP_NO_USER_LOW + m_rec_no)
+				 | ulint(m_is_comp) << 15,
+				 MLOG_2BYTES, &m_mtr);
+		mlog_write_ulint(PAGE_HEADER + PAGE_N_RECS + m_page, m_rec_no,
+				 MLOG_2BYTES, &m_mtr);
+		mlog_write_ulint(PAGE_HEADER + PAGE_LAST_INSERT + m_page,
+				 ulint(m_cur_rec - m_page),
+				 MLOG_2BYTES, &m_mtr);
+		mlog_write_ulint(PAGE_HEADER + PAGE_DIRECTION + m_page,
+				 PAGE_RIGHT, MLOG_2BYTES, &m_mtr);
+		mlog_write_ulint(PAGE_HEADER + PAGE_N_DIRECTION + m_page, 0,
+				 MLOG_2BYTES, &m_mtr);
+	} else {
+		/* For ROW_FORMAT=COMPRESSED, redo log may be written
+		in PageBulk::compress(). */
+		mach_write_to_2(PAGE_HEADER + PAGE_N_DIR_SLOTS + m_page,
+				2 + slot_index);
+		mach_write_to_2(PAGE_HEADER + PAGE_HEAP_TOP + m_page,
+				ulint(m_heap_top - m_page));
+		mach_write_to_2(PAGE_HEADER + PAGE_N_HEAP + m_page,
+				(PAGE_HEAP_NO_USER_LOW + m_rec_no)
+				| ulint(m_is_comp) << 15);
+		mach_write_to_2(PAGE_HEADER + PAGE_N_RECS + m_page, m_rec_no);
+		mach_write_to_2(PAGE_HEADER + PAGE_LAST_INSERT + m_page,
+				ulint(m_cur_rec - m_page));
+		mach_write_to_2(PAGE_HEADER + PAGE_DIRECTION + m_page,
+				PAGE_RIGHT);
+		mach_write_to_2(PAGE_HEADER + PAGE_N_DIRECTION + m_page, 0);
+	}
 
 	m_block->skip_flush_check = false;
 }
@@ -470,20 +523,30 @@ PageBulk::copyOut(
 
 /** Set next page
 @param[in]	next_page_no	next page no */
-void
-PageBulk::setNext(
-	ulint		next_page_no)
+inline void PageBulk::setNext(ulint next_page_no)
 {
-	btr_page_set_next(m_page, NULL, next_page_no, &m_mtr);
+	if (UNIV_LIKELY_NULL(m_page_zip)) {
+		/* For ROW_FORMAT=COMPRESSED, redo log may be written
+		in PageBulk::compress(). */
+		mach_write_to_4(m_page + FIL_PAGE_NEXT, next_page_no);
+	} else {
+		mlog_write_ulint(m_page + FIL_PAGE_NEXT, next_page_no,
+				 MLOG_4BYTES, &m_mtr);
+	}
 }
 
 /** Set previous page
 @param[in]	prev_page_no	previous page no */
-void
-PageBulk::setPrev(
-	ulint		prev_page_no)
+inline void PageBulk::setPrev(ulint prev_page_no)
 {
-	btr_page_set_prev(m_page, NULL, prev_page_no, &m_mtr);
+	if (UNIV_LIKELY_NULL(m_page_zip)) {
+		/* For ROW_FORMAT=COMPRESSED, redo log may be written
+		in PageBulk::compress(). */
+		mach_write_to_4(m_page + FIL_PAGE_PREV, prev_page_no);
+	} else {
+		mlog_write_ulint(m_page + FIL_PAGE_PREV, prev_page_no,
+				 MLOG_4BYTES, &m_mtr);
+	}
 }
 
 /** Check if required space is available in the page for the rec to be inserted.
@@ -591,8 +654,12 @@ PageBulk::latch()
 {
 	m_mtr.start();
 	mtr_x_lock(&m_index->lock, &m_mtr);
-	m_mtr.set_log_mode(MTR_LOG_NO_REDO);
-	m_mtr.set_flush_observer(m_flush_observer);
+	if (m_flush_observer) {
+		m_mtr.set_log_mode(MTR_LOG_NO_REDO);
+		m_mtr.set_flush_observer(m_flush_observer);
+	} else {
+		m_mtr.set_named_space(m_index->space);
+	}
 
 	/* In case the block is S-latched by page_cleaner. */
 	if (!buf_page_optimistic_get(RW_X_LATCH, m_block, m_modify_clock,
@@ -635,7 +702,7 @@ BtrBulk::pageSplit(
 	}
 
 	/* 2. create a new page. */
-	PageBulk new_page_bulk(m_index, m_trx_id, FIL_NULL,
+	PageBulk new_page_bulk(m_index, m_trx->id, FIL_NULL,
 			       page_bulk->getLevel(), m_flush_observer);
 	dberr_t	err = new_page_bulk.init();
 	if (err != DB_SUCCESS) {
@@ -714,8 +781,7 @@ BtrBulk::pageCommit(
 }
 
 /** Log free check */
-void
-BtrBulk::logFreeCheck()
+inline void BtrBulk::logFreeCheck()
 {
 	if (log_sys->check_flush_or_checkpoint) {
 		release();
@@ -766,7 +832,7 @@ BtrBulk::insert(
 	/* Check if we need to create a PageBulk for the level. */
 	if (level + 1 > m_page_bulks.size()) {
 		PageBulk*	new_page_bulk
-			= UT_NEW_NOKEY(PageBulk(m_index, m_trx_id, FIL_NULL,
+			= UT_NEW_NOKEY(PageBulk(m_index, m_trx->id, FIL_NULL,
 						level, m_flush_observer));
 		err = new_page_bulk->init();
 		if (err != DB_SUCCESS) {
@@ -819,7 +885,7 @@ BtrBulk::insert(
 	if (!page_bulk->isSpaceAvailable(rec_size)) {
 		/* Create a sibling page_bulk. */
 		PageBulk*	sibling_page_bulk;
-		sibling_page_bulk = UT_NEW_NOKEY(PageBulk(m_index, m_trx_id,
+		sibling_page_bulk = UT_NEW_NOKEY(PageBulk(m_index, m_trx->id,
 							  FIL_NULL, level,
 							  m_flush_observer));
 		err = sibling_page_bulk->init();
@@ -845,8 +911,11 @@ BtrBulk::insert(
 
 		/* Important: log_free_check whether we need a checkpoint. */
 		if (page_is_leaf(sibling_page_bulk->getPage())) {
-			/* Check whether trx is interrupted */
-			if (m_flush_observer->check_interrupted()) {
+			if (trx_is_interrupted(m_trx)) {
+				if (m_flush_observer) {
+					m_flush_observer->interrupted();
+				}
+
 				err = DB_INTERRUPTED;
 				goto func_exit;
 			}
@@ -944,7 +1013,7 @@ BtrBulk::finish(dberr_t	err)
 					last_page_no);
 		page_size_t	page_size(dict_table_page_size(m_index->table));
 		ulint		root_page_no = dict_index_get_page(m_index);
-		PageBulk	root_page_bulk(m_index, m_trx_id,
+		PageBulk	root_page_bulk(m_index, m_trx->id,
 					       root_page_no, m_root_level,
 					       m_flush_observer);
 
diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc
index cf53a10fa21..75a0498dde5 100644
--- a/storage/innobase/buf/buf0flu.cc
+++ b/storage/innobase/buf/buf0flu.cc
@@ -3735,18 +3735,12 @@ FlushObserver::~FlushObserver()
 	DBUG_LOG("flush", "~FlushObserver(): trx->id=" << m_trx->id);
 }
 
-/** Check whether trx is interrupted
-@return true if trx is interrupted */
-bool
-FlushObserver::check_interrupted()
+/** Check whether the operation has been interrupted */
+void FlushObserver::check_interrupted()
 {
 	if (trx_is_interrupted(m_trx)) {
 		interrupted();
-
-		return(true);
 	}
-
-	return(false);
 }
 
 /** Notify observer of a flush
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index 7000865a7cd..a8594fd37b3 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -20359,6 +20359,13 @@ static MYSQL_SYSVAR_BOOL(log_compressed_pages, page_zip_log_pages,
   " compression algorithm doesn't change.",
   NULL, NULL, TRUE);
 
+static MYSQL_SYSVAR_BOOL(log_optimize_ddl, innodb_log_optimize_ddl,
+  PLUGIN_VAR_OPCMDARG,
+  "Reduce redo logging when natively creating indexes or rebuilding tables."
+  " Setting this OFF avoids delay due to page flushing and"
+  " allows concurrent backup.",
+  NULL, NULL, TRUE);
+
 static MYSQL_SYSVAR_ULONG(autoextend_increment,
   sys_tablespace_auto_extend_increment,
   PLUGIN_VAR_RQCMDARG,
@@ -21278,6 +21285,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
   MYSQL_SYSVAR(log_write_ahead_size),
   MYSQL_SYSVAR(log_group_home_dir),
   MYSQL_SYSVAR(log_compressed_pages),
+  MYSQL_SYSVAR(log_optimize_ddl),
   MYSQL_SYSVAR(max_dirty_pages_pct),
   MYSQL_SYSVAR(max_dirty_pages_pct_lwm),
   MYSQL_SYSVAR(adaptive_flushing_lwm),
diff --git a/storage/innobase/include/btr0bulk.h b/storage/innobase/include/btr0bulk.h
index a3af0ea7c93..edf34bd5ae4 100644
--- a/storage/innobase/include/btr0bulk.h
+++ b/storage/innobase/include/btr0bulk.h
@@ -34,6 +34,8 @@ Created 03/11/2014 Shaohua Wang
 
 /** Innodb B-tree index fill factor for bulk load. */
 extern	long	innobase_fill_factor;
+/** whether to reduce redo logging during ALTER TABLE */
+extern	my_bool	innodb_log_optimize_ddl;
 
 /*
 The proper function call sequence of PageBulk is as below:
@@ -146,11 +148,11 @@ public:
 
 	/** Set next page
 	@param[in]	next_page_no	next page no */
-	void setNext(ulint	next_page_no);
+	inline void setNext(ulint next_page_no);
 
 	/** Set previous page
 	@param[in]	prev_page_no	previous page no */
-	void setPrev(ulint	prev_page_no);
+	inline void setPrev(ulint prev_page_no);
 
 	/** Release block by commiting mtr */
 	inline void release();
@@ -257,7 +259,7 @@ private:
 	when the block is re-pinned */
 	ib_uint64_t     m_modify_clock;
 
-	/** Flush observer */
+	/** Flush observer, or NULL if redo logging is enabled */
 	FlushObserver*	m_flush_observer;
 
 	/** Operation result DB_SUCCESS or error code */
@@ -272,19 +274,19 @@ class BtrBulk
 public:
 	/** Constructor
 	@param[in]	index		B-tree index
-	@param[in]	trx_id		transaction id
+	@param[in]	trx		transaction
 	@param[in]	observer	flush observer */
 	BtrBulk(
 		dict_index_t*	index,
-		trx_id_t	trx_id,
+		const trx_t*	trx,
 		FlushObserver*	observer)
 		:
 		m_index(index),
-		m_trx_id(trx_id),
+		m_trx(trx),
 		m_flush_observer(observer)
 	{
-		ut_ad(m_flush_observer != NULL);
 #ifdef UNIV_DEBUG
+		if (m_flush_observer)
 		fil_space_inc_redo_skipped_count(m_index->space);
 #endif /* UNIV_DEBUG */
 	}
@@ -293,6 +295,7 @@ public:
 	~BtrBulk()
 	{
 #ifdef UNIV_DEBUG
+		if (m_flush_observer)
 		fil_space_dec_redo_skipped_count(m_index->space);
 #endif /* UNIV_DEBUG */
 	}
@@ -354,20 +357,20 @@ private:
 	}
 
 	/** Log free check */
-	void logFreeCheck();
+	inline void logFreeCheck();
 
 private:
 	/** B-tree index */
-	dict_index_t*		m_index;
+	dict_index_t*const	m_index;
 
-	/** Transaction id */
-	trx_id_t		m_trx_id;
+	/** Transaction */
+	const trx_t*const	m_trx;
 
 	/** Root page level */
 	ulint			m_root_level;
 
-	/** Flush observer */
-	FlushObserver*		m_flush_observer;
+	/** Flush observer, or NULL if redo logging is enabled */
+	FlushObserver*const	m_flush_observer;
 
 	/** Page cursor vector for all level */
 	page_bulk_vector	m_page_bulks;
diff --git a/storage/innobase/include/buf0flu.h b/storage/innobase/include/buf0flu.h
index c3b518c4295..5c1dddd9a3b 100644
--- a/storage/innobase/include/buf0flu.h
+++ b/storage/innobase/include/buf0flu.h
@@ -371,9 +371,8 @@ public:
 		m_interrupted = true;
 	}
 
-	/** Check whether trx is interrupted
-	@return true if trx is interrupted */
-	bool check_interrupted();
+	/** Check whether the operation has been interrupted */
+	void check_interrupted();
 
 	/** Flush dirty pages. */
 	void flush();
@@ -395,7 +394,7 @@ private:
 	const ulint		m_space_id;
 
 	/** Trx instance */
-	trx_t* const		m_trx;
+	const trx_t* const	m_trx;
 
 	/** Performance schema accounting object, used by ALTER TABLE.
 	If not NULL, then stage->begin_phase_flush() will be called initially,
diff --git a/storage/innobase/include/page0cur.h b/storage/innobase/include/page0cur.h
index f3890c43b73..a038f68731c 100644
--- a/storage/innobase/include/page0cur.h
+++ b/storage/innobase/include/page0cur.h
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -330,6 +331,20 @@ page_cur_open_on_rnd_user_rec(
 /*==========================*/
 	buf_block_t*	block,	/*!< in: page */
 	page_cur_t*	cursor);/*!< out: page cursor */
+/** Write a redo log record of inserting a record into an index page.
+@param[in]	insert_rec	inserted record
+@param[in]	rec_size	rec_get_size(insert_rec)
+@param[in]	cursor_rec	predecessor of insert_rec
+@param[in,out]	index		index tree
+@param[in,out]	mtr		mini-transaction */
+void
+page_cur_insert_rec_write_log(
+	const rec_t*	insert_rec,
+	ulint		rec_size,
+	const rec_t*	cursor_rec,
+	dict_index_t*	index,
+	mtr_t*		mtr)
+	MY_ATTRIBUTE((nonnull));
 /***********************************************************//**
 Parses a log record of a record insert on a page.
 @return end of log record or NULL */
diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc
index 113c31e44ae..b611d484306 100644
--- a/storage/innobase/page/page0cur.cc
+++ b/storage/innobase/page/page0cur.cc
@@ -2,6 +2,7 @@
 
 Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2012, Facebook Inc.
+Copyright (c) 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -834,18 +835,19 @@ page_cur_open_on_rnd_user_rec(
 	} while (rnd--);
 }
 
-/***********************************************************//**
-Writes the log record of a record insert on a page. */
-static
+/** Write a redo log record of inserting a record into an index page.
+@param[in]	insert_rec	inserted record
+@param[in]	rec_size	rec_get_size(insert_rec)
+@param[in]	cursor_rec	predecessor of insert_rec
+@param[in,out]	index		index tree
+@param[in,out]	mtr		mini-transaction */
 void
 page_cur_insert_rec_write_log(
-/*==========================*/
-	rec_t*		insert_rec,	/*!< in: inserted physical record */
-	ulint		rec_size,	/*!< in: insert_rec size */
-	rec_t*		cursor_rec,	/*!< in: record the
-					cursor is pointing to */
-	dict_index_t*	index,		/*!< in: record descriptor */
-	mtr_t*		mtr)		/*!< in: mini-transaction handle */
+	const rec_t*	insert_rec,
+	ulint		rec_size,
+	const rec_t*	cursor_rec,
+	dict_index_t*	index,
+	mtr_t*		mtr)
 {
 	ulint	cur_rec_size;
 	ulint	extra_size;
diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc
index c59d3a95e3e..6abc84ddee7 100644
--- a/storage/innobase/row/row0ftsort.cc
+++ b/storage/innobase/row/row0ftsort.cc
@@ -1684,11 +1684,10 @@ row_fts_merge_insert(
 	dict_table_close(aux_table, FALSE, FALSE);
 	aux_index = dict_table_get_first_index(aux_table);
 
-	FlushObserver* observer;
-	observer = psort_info[0].psort_common->trx->flush_observer;
-
 	/* Create bulk load instance */
-	ins_ctx.btr_bulk = UT_NEW_NOKEY(BtrBulk(aux_index, trx->id, observer));
+	ins_ctx.btr_bulk = UT_NEW_NOKEY(
+		BtrBulk(aux_index, trx, psort_info[0].psort_common->trx
+			->flush_observer));
 
 	/* Create tuple for insert */
 	ins_ctx.tuple = dtuple_create(heap, dict_index_get_n_fields(aux_index));
diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc
index 8edea4959f1..951c16394a7 100644
--- a/storage/innobase/row/row0merge.cc
+++ b/storage/innobase/row/row0merge.cc
@@ -274,7 +274,6 @@ private:
 #define FTS_PENDING_DOC_MEMORY_LIMIT	1000000
 
 /** Insert sorted data tuples to the index.
-@param[in]	trx_id		transaction identifier
 @param[in]	index		index to be inserted
 @param[in]	old_table	old table
 @param[in]	fd		file descriptor
@@ -289,7 +288,6 @@ and then stage->inc() will be called for each record that is processed.
 static	MY_ATTRIBUTE((warn_unused_result))
 dberr_t
 row_merge_insert_index_tuples(
-	trx_id_t		trx_id,
 	dict_index_t*		index,
 	const dict_table_t*	old_table,
 	int			fd,
@@ -2402,14 +2400,14 @@ write_buffers:
 					if (clust_btr_bulk == NULL) {
 						clust_btr_bulk = UT_NEW_NOKEY(
 							BtrBulk(index[i],
-								trx->id,
-								observer));
+								trx,
+								observer/**/));
 					} else {
 						clust_btr_bulk->latch();
 					}
 
 					err = row_merge_insert_index_tuples(
-						trx->id, index[i], old_table,
+						index[i], old_table,
 						-1, NULL, buf, clust_btr_bulk,
 						table_total_rows,
 						curr_progress,
@@ -2515,11 +2513,11 @@ write_buffers:
 						trx->error_key_num = i;
 						goto all_done;);
 
-					BtrBulk	btr_bulk(index[i], trx->id,
+					BtrBulk	btr_bulk(index[i], trx,
 							 observer);
 
 					err = row_merge_insert_index_tuples(
-						trx->id, index[i], old_table,
+						index[i], old_table,
 						-1, NULL, buf, &btr_bulk,
 						table_total_rows,
 						curr_progress,
@@ -3375,7 +3373,6 @@ row_merge_mtuple_to_dtuple(
 }
 
 /** Insert sorted data tuples to the index.
-@param[in]	trx_id		transaction identifier
 @param[in]	index		index to be inserted
 @param[in]	old_table	old table
 @param[in]	fd		file descriptor
@@ -3390,7 +3387,6 @@ and then stage->inc() will be called for each record that is processed.
 static	MY_ATTRIBUTE((warn_unused_result))
 dberr_t
 row_merge_insert_index_tuples(
-	trx_id_t		trx_id,
 	dict_index_t*		index,
 	const dict_table_t*	old_table,
 	int			fd,
@@ -3428,7 +3424,6 @@ row_merge_insert_index_tuples(
 	ut_ad(!srv_read_only_mode);
 	ut_ad(!(index->type & DICT_FTS));
 	ut_ad(!dict_index_is_spatial(index));
-	ut_ad(trx_id);
 
 	if (stage != NULL) {
 		stage->begin_phase_insert();
@@ -4632,11 +4627,15 @@ row_merge_build_indexes(
 	we use bulk load to create all types of indexes except spatial index,
 	for which redo logging is enabled. If we create only spatial indexes,
 	we don't need to flush dirty pages at all. */
-	bool	need_flush_observer = (old_table != new_table);
+	bool	need_flush_observer = bool(innodb_log_optimize_ddl);
 
-	for (i = 0; i < n_indexes; i++) {
-		if (!dict_index_is_spatial(indexes[i])) {
-			need_flush_observer = true;
+	if (need_flush_observer) {
+		need_flush_observer = old_table != new_table;
+
+		for (i = 0; i < n_indexes; i++) {
+			if (!dict_index_is_spatial(indexes[i])) {
+				need_flush_observer = true;
+			}
 		}
 	}
 
@@ -4883,7 +4882,7 @@ wait_again:
 				os_thread_sleep(20000000););  /* 20 sec */
 
 			if (error == DB_SUCCESS) {
-				BtrBulk	btr_bulk(sort_idx, trx->id,
+				BtrBulk	btr_bulk(sort_idx, trx,
 						 flush_observer);
 
 				pct_cost = (COST_BUILD_INDEX_STATIC +
@@ -4903,7 +4902,7 @@ wait_again:
 				}
 
 				error = row_merge_insert_index_tuples(
-					trx->id, sort_idx, old_table,
+					sort_idx, old_table,
 					merge_files[i].fd, block, NULL,
 					&btr_bulk,
 					merge_files[i].n_rec, pct_progress, pct_cost,
@@ -4936,14 +4935,16 @@ wait_again:
 			ut_ad(sort_idx->online_status
 			      == ONLINE_INDEX_COMPLETE);
 		} else {
-			ut_ad(need_flush_observer);
+			if (flush_observer) {
+				flush_observer->flush();
+				row_merge_write_redo(indexes[i]);
+			}
+
 			if (global_system_variables.log_warnings > 2) {
 				sql_print_information(
 					"InnoDB: Online DDL : Applying"
 					" log to index");
 			}
-			flush_observer->flush();
-			row_merge_write_redo(indexes[i]);
 
 			DEBUG_SYNC_C("row_log_apply_before");
 			error = row_log_apply(trx, sort_idx, table, stage);

From 3c141e319ab29afdfe843553da385fe5d981906e Mon Sep 17 00:00:00 2001
From: Igor Babaev <igor@askmonty.org>
Date: Wed, 25 Jul 2018 23:28:31 -0700
Subject: [PATCH 069/164] MDEV-15087 Item_func::fix_fields:           
 Assertion `used_tables_cache == 0' failed

This bug manifested itself when executing queries
over materialized derived tables /vies and with
conjunctive always true predicates containing
inexpensive single-row subqueries.
This bug disappeared after the patch mdev-15035
had been applied.
---
 mysql-test/r/derived_cond_pushdown.result | 19 +++++++++++++++++
 mysql-test/t/derived_cond_pushdown.test   | 25 +++++++++++++++++++++++
 sql/sql_select.cc                         | 14 ++++++-------
 3 files changed, 51 insertions(+), 7 deletions(-)

diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result
index a2e84a1e419..3adfe0b9b01 100644
--- a/mysql-test/r/derived_cond_pushdown.result
+++ b/mysql-test/r/derived_cond_pushdown.result
@@ -9808,3 +9808,22 @@ EXPLAIN
   }
 }
 DROP TABLE t1;
+#
+# MDEV-15087: error from inexpensive subquery before check
+#                   for condition pushdown into derived 
+#
+CREATE TABLE t1 (i1 int, v1 varchar(1));
+INSERT INTO t1 VALUES (7,'x');
+CREATE TABLE t2 (i1 int);
+INSERT INTO t2 VALUES (8);
+CREATE TABLE t3 (i1 int ,v1 varchar(1), v2 varchar(1));
+INSERT INTO t3 VALUES (4, 'v','v'),(62,'v','k'),(7, 'n', NULL);
+SELECT 1
+FROM (t1 AS a1
+JOIN (((SELECT  DISTINCT t3.*
+FROM t3) AS a2
+JOIN t1 ON (t1.v1 = a2.v2))) ON (t1.v1 = a2.v1))
+WHERE (SELECT  BIT_COUNT(t2.i1)
+FROM (t2 JOIN t3)) IS NULL;
+ERROR 21000: Subquery returns more than 1 row
+DROP TABLE t1, t2, t3;
diff --git a/mysql-test/t/derived_cond_pushdown.test b/mysql-test/t/derived_cond_pushdown.test
index 3103ddb5c55..872acb55589 100644
--- a/mysql-test/t/derived_cond_pushdown.test
+++ b/mysql-test/t/derived_cond_pushdown.test
@@ -1840,3 +1840,28 @@ EVAL $query;
 EVAL EXPLAIN FORMAT=JSON $query;
 
 DROP TABLE t1;
+
+--echo #
+--echo # MDEV-15087: error from inexpensive subquery before check
+--echo #                   for condition pushdown into derived 
+--echo #
+
+CREATE TABLE t1 (i1 int, v1 varchar(1));
+INSERT INTO t1 VALUES (7,'x');
+
+CREATE TABLE t2 (i1 int);
+INSERT INTO t2 VALUES (8);
+
+CREATE TABLE t3 (i1 int ,v1 varchar(1), v2 varchar(1));
+INSERT INTO t3 VALUES (4, 'v','v'),(62,'v','k'),(7, 'n', NULL);
+
+--error ER_SUBQUERY_NO_1_ROW
+SELECT 1
+FROM (t1 AS a1
+      JOIN (((SELECT  DISTINCT t3.*
+              FROM t3) AS a2
+             JOIN t1 ON (t1.v1 = a2.v2))) ON (t1.v1 = a2.v1))
+WHERE (SELECT  BIT_COUNT(t2.i1)
+       FROM (t2 JOIN t3)) IS NULL;
+
+DROP TABLE t1, t2, t3;
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 4cabf5c1b40..47ca733468b 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -1368,6 +1368,13 @@ JOIN::optimize_inner()
   conds= optimize_cond(this, conds, join_list, FALSE,
                        &cond_value, &cond_equal, OPT_LINK_EQUAL_FIELDS);
   
+  if (thd->is_error())
+  {
+    error= 1;
+    DBUG_PRINT("error",("Error from optimize_cond"));
+    DBUG_RETURN(1);
+  }
+
   if (thd->lex->sql_command == SQLCOM_SELECT &&
       optimizer_flag(thd, OPTIMIZER_SWITCH_COND_PUSHDOWN_FOR_DERIVED))
   {
@@ -1402,13 +1409,6 @@ JOIN::optimize_inner()
       DBUG_RETURN(1);
   }
      
-  if (thd->is_error())
-  {
-    error= 1;
-    DBUG_PRINT("error",("Error from optimize_cond"));
-    DBUG_RETURN(1);
-  }
-
   {
     having= optimize_cond(this, having, join_list, TRUE,
                           &having_value, &having_equal);

From a97c190d95bc9f3be0c0fa75abe56422927e7d82 Mon Sep 17 00:00:00 2001
From: Andrei Elkin <andrei.elkin@mariadb.com>
Date: Tue, 24 Jul 2018 18:01:30 +0300
Subject: [PATCH 070/164] MDEV-16812 Semisync slave io thread segfaults at
 STOP-SLAVE handling

When the semisync slave is being stopped with STOP SLAVE just after
the master was shut down it attempts to reconnect with the master
anyway per a semisync routine.  Instead of an expected error the
io-thread segfauls in mysql_real_connect() execution at

 !mysql->options.extension->async_context

check trying to reach the extension's member while mysql->options.extension is
actually and correctly NULL.
Apparently not-NULL check for mysql->options.extension was missed and
it's deployed by the patch to fix this issue.

As a bonus it also tackles an assert
   Thread 0x7f16c72148c0 (LWP 24639) 0x00007f16c53b3bf2 in __GI___assert_fail (assertion=0x55a686117558 "global_status_var.global_memory_used == 0", file=0x55a6861171e8 "/home/andrei/MDB/WTs/10.3-clean/sql/mysqld.cc", line=2201, function=0x55a68611fa80 <mysqld_exit(int)::__PRETTY_FUNCTION__> "void mysqld_exit(int)") at assert.c:101

in a new test of the patch. The reason of the assert was insufficient cleanup
in Repl_semi_sync_slave::kill_connection() which has a branch where a MYSQL instance
was left out unfred.
---
 .../r/rpl_semi_sync_master_shutdown.result    | 33 ++++++++++
 .../rpl/t/rpl_semi_sync_master_shutdown.test  | 60 +++++++++++++++++++
 sql-common/client.c                           |  2 +-
 sql/semisync_slave.cc                         |  3 +-
 4 files changed, 95 insertions(+), 3 deletions(-)
 create mode 100644 mysql-test/suite/rpl/r/rpl_semi_sync_master_shutdown.result
 create mode 100644 mysql-test/suite/rpl/t/rpl_semi_sync_master_shutdown.test

diff --git a/mysql-test/suite/rpl/r/rpl_semi_sync_master_shutdown.result b/mysql-test/suite/rpl/r/rpl_semi_sync_master_shutdown.result
new file mode 100644
index 00000000000..786e1682bb0
--- /dev/null
+++ b/mysql-test/suite/rpl/r/rpl_semi_sync_master_shutdown.result
@@ -0,0 +1,33 @@
+include/master-slave.inc
+[connection master]
+connection master;
+SET @@GLOBAL.rpl_semi_sync_master_enabled = 1;
+connection slave;
+include/stop_slave.inc
+SET @@GLOBAL. rpl_semi_sync_slave_enabled = 1;
+include/start_slave.inc
+connection master;
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 SET a=1;
+connection slave;
+connection master;
+# Shutdown master
+include/rpl_stop_server.inc [server_number=1]
+connection slave;
+include/stop_slave.inc
+# Restart master
+include/rpl_start_server.inc [server_number=1]
+connection slave;
+include/stop_slave.inc
+Warnings:
+Note	1255	Slave already has been stopped
+include/start_slave.inc
+connection master;
+SET @@GLOBAL.debug_dbug="";
+SET @@GLOBAL. rpl_semi_sync_master_enabled = 0;
+connection master;
+DROP TABLE t1;
+connection slave;
+include/stop_slave.inc
+SET @@GLOBAL. rpl_semi_sync_slave_enabled = 0;
+include/rpl_end.inc
diff --git a/mysql-test/suite/rpl/t/rpl_semi_sync_master_shutdown.test b/mysql-test/suite/rpl/t/rpl_semi_sync_master_shutdown.test
new file mode 100644
index 00000000000..2224f78d6d0
--- /dev/null
+++ b/mysql-test/suite/rpl/t/rpl_semi_sync_master_shutdown.test
@@ -0,0 +1,60 @@
+# MDEV-16812 Semisync slave io thread segfaults at STOP-SLAVE handling
+#
+# The test verifies that the semisync-enabled slave io thread
+# finishes off as specified in particular trying to connect even to a shut down
+# master for a semisync firewell routine.
+
+source include/not_embedded.inc;
+source include/have_debug.inc;
+source include/master-slave.inc;
+
+--connection master
+
+--let $sav_enabled_master=`SELECT @@GLOBAL.rpl_semi_sync_master_enabled `
+SET @@GLOBAL.rpl_semi_sync_master_enabled = 1;
+
+--connection slave
+source include/stop_slave.inc;
+--let $sav_enabled_slave=`SELECT @@GLOBAL.rpl_semi_sync_slave_enabled `
+SET @@GLOBAL. rpl_semi_sync_slave_enabled = 1;
+source include/start_slave.inc;
+
+--connection master
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 SET a=1;
+
+--sync_slave_with_master
+
+connection master;
+--echo # Shutdown master
+--let $rpl_server_number=1
+source include/rpl_stop_server.inc;
+
+--connection slave
+--source include/stop_slave.inc
+
+#connection master;
+--echo # Restart master
+--let $rpl_server_number=1
+source include/rpl_start_server.inc;
+
+#
+# Clean up
+#
+--connection slave
+--source include/stop_slave.inc
+--source include/start_slave.inc
+
+--connection master
+SET @@GLOBAL.debug_dbug="";
+--eval SET @@GLOBAL. rpl_semi_sync_master_enabled = $sav_enabled_master
+
+--connection master
+DROP TABLE t1;
+
+--sync_slave_with_master
+source include/stop_slave.inc;
+--eval SET @@GLOBAL. rpl_semi_sync_slave_enabled = $sav_enabled_slave
+
+--let $rpl_only_running_threads= 1
+--source include/rpl_end.inc
diff --git a/sql-common/client.c b/sql-common/client.c
index 088377f8c52..8bbe08a504a 100644
--- a/sql-common/client.c
+++ b/sql-common/client.c
@@ -3657,7 +3657,7 @@ error:
     end_server(mysql);
     mysql_close_free(mysql);
     if (!(client_flag & CLIENT_REMEMBER_OPTIONS) &&
-        !mysql->options.extension->async_context)
+        !(mysql->options.extension && mysql->options.extension->async_context))
       mysql_close_free_options(mysql);
   }
   DBUG_RETURN(0);
diff --git a/sql/semisync_slave.cc b/sql/semisync_slave.cc
index 86d0176dac1..df8baf045ac 100644
--- a/sql/semisync_slave.cc
+++ b/sql/semisync_slave.cc
@@ -144,8 +144,7 @@ void Repl_semi_sync_slave::kill_connection(MYSQL *mysql)
   {
     sql_print_information("cannot connect to master to kill slave io_thread's "
                           "connection");
-    if (!ret)
-      mysql_close(kill_mysql);
+    mysql_close(kill_mysql);
     return;
   }
   size_t kill_buffer_length = my_snprintf(kill_buffer, 30, "KILL %lu",

From 5c5a116b47020e7ed79a59d4d2ced69a686ff090 Mon Sep 17 00:00:00 2001
From: Alexander Barkov <bar@mariadb.com>
Date: Thu, 26 Jul 2018 12:04:21 +0400
Subject: [PATCH 071/164] MDEV-16614 signal 7 after calling stored procedure,
 that uses regexp

The problem happened in the derived condition pushdown code:
- When Item_func_regex::build_clone() was called, it created a copy of
  the original Item_func_regex, and this copy got registered in free_list.
  Class specific additional dynamic members (such as "re") made
  a shallow copy, rather than a deep copy, in the cloned Item_func_regex.
  As a result, the Regexp_processor_pcre::m_pcre of the cloned Item_func_regex
  and of the original Item_func_regex pointed to the same compiled regular
  expression.
- On cleanup_items(), both original and cloned copies of Item_func_regex
  called re.cleanup(), which called pcre_free(m_pcre). So the same compiled
  regular expression was freed two times, which was noticed by ASAN.

The same problem was repeatable for Item_func_regexp_instr.

A similar problem happened for Item_func_sp, for the sp_result_field member.
Both original and cloned copies of Item_func_sp pointed the same Field instance
and both deleted it on cleanup().

A possible solution would be to fix build_clone() to create deep
(instead of shallow) copies for the dynamic members of the affected classes
(Item_func_regex, Item_func_regexp_instr, Item_func sp).
However, this would be too complex.

As agreed with Galina and Igor, this patch disallows using using these
affected classes in derived condition pushdown by overriding get_clone()
to return NULL.
---
 mysql-test/r/derived_cond_pushdown.result | 47 +++++++++++++++++++
 mysql-test/t/derived_cond_pushdown.test   | 55 +++++++++++++++++++++++
 sql/item_cmpfunc.h                        | 14 +-----
 sql/item_func.h                           | 10 +----
 4 files changed, 105 insertions(+), 21 deletions(-)

diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result
index 3adfe0b9b01..22b81ef48a5 100644
--- a/mysql-test/r/derived_cond_pushdown.result
+++ b/mysql-test/r/derived_cond_pushdown.result
@@ -9827,3 +9827,50 @@ WHERE (SELECT  BIT_COUNT(t2.i1)
 FROM (t2 JOIN t3)) IS NULL;
 ERROR 21000: Subquery returns more than 1 row
 DROP TABLE t1, t2, t3;
+#
+# MDEV-16614 signal 7 after calling stored procedure, that uses regexp
+#
+CREATE PROCEDURE p1(m1 varchar(5), m2 varchar(5))
+BEGIN
+SELECT a FROM
+(SELECT "aa" a) t
+JOIN (SELECT "aa" b) t1 on t.a=t1.b
+WHERE t.a regexp m1 and t1.b regexp m2
+GROUP BY a;
+END$$
+CALL p1('a','a');
+a
+aa
+DROP PROCEDURE p1;
+CREATE PROCEDURE p1(m1 varchar(5))
+BEGIN
+SELECT a FROM (SELECT "aa" a) t WHERE t.a regexp m1;
+END$$
+CALL p1('a');
+a
+aa
+DROP PROCEDURE p1;
+SELECT a FROM (SELECT "aa" a) t WHERE REGEXP_INSTR(t.a, (SELECT MAX('aa') FROM DUAL LIMIT 1));
+a
+aa
+CREATE OR REPLACE FUNCTION f1(a VARCHAR(10), b VARCHAR(10)) RETURNS INT
+BEGIN
+RETURN 1;
+END;$$
+CREATE OR REPLACE PROCEDURE p1(m1 varchar(5))
+BEGIN
+SELECT a FROM (SELECT "aa" a) t WHERE f1(t.a, m1);
+END$$
+CALL p1('a');
+a
+aa
+DROP PROCEDURE p1;
+DROP FUNCTION f1;
+CREATE OR REPLACE FUNCTION f1(a VARCHAR(10), b VARCHAR(10)) RETURNS INT
+BEGIN
+RETURN 1;
+END;$$
+SELECT a FROM (SELECT "aa" a) t WHERE f1(t.a, (SELECT MAX('aa') FROM DUAL LIMIT 1));
+a
+aa
+DROP FUNCTION f1;
diff --git a/mysql-test/t/derived_cond_pushdown.test b/mysql-test/t/derived_cond_pushdown.test
index 872acb55589..d9d767f53ed 100644
--- a/mysql-test/t/derived_cond_pushdown.test
+++ b/mysql-test/t/derived_cond_pushdown.test
@@ -1865,3 +1865,58 @@ WHERE (SELECT  BIT_COUNT(t2.i1)
        FROM (t2 JOIN t3)) IS NULL;
 
 DROP TABLE t1, t2, t3;
+
+--echo #
+--echo # MDEV-16614 signal 7 after calling stored procedure, that uses regexp
+--echo #
+
+DELIMITER $$;
+CREATE PROCEDURE p1(m1 varchar(5), m2 varchar(5))
+BEGIN
+SELECT a FROM
+	(SELECT "aa" a) t
+	JOIN (SELECT "aa" b) t1 on t.a=t1.b
+WHERE t.a regexp m1 and t1.b regexp m2
+GROUP BY a;
+END$$
+DELIMITER ;$$
+CALL p1('a','a');
+DROP PROCEDURE p1;
+
+
+DELIMITER $$;
+CREATE PROCEDURE p1(m1 varchar(5))
+BEGIN
+  SELECT a FROM (SELECT "aa" a) t WHERE t.a regexp m1;
+END$$
+DELIMITER ;$$
+CALL p1('a');
+DROP PROCEDURE p1;
+
+
+SELECT a FROM (SELECT "aa" a) t WHERE REGEXP_INSTR(t.a, (SELECT MAX('aa') FROM DUAL LIMIT 1));
+
+
+DELIMITER $$;
+CREATE OR REPLACE FUNCTION f1(a VARCHAR(10), b VARCHAR(10)) RETURNS INT
+BEGIN
+  RETURN 1;
+END;$$
+CREATE OR REPLACE PROCEDURE p1(m1 varchar(5))
+BEGIN
+  SELECT a FROM (SELECT "aa" a) t WHERE f1(t.a, m1);
+END$$
+DELIMITER ;$$
+CALL p1('a');
+DROP PROCEDURE p1;
+DROP FUNCTION f1;
+
+
+DELIMITER $$;
+CREATE OR REPLACE FUNCTION f1(a VARCHAR(10), b VARCHAR(10)) RETURNS INT
+BEGIN
+  RETURN 1;
+END;$$
+DELIMITER ;$$
+SELECT a FROM (SELECT "aa" a) t WHERE f1(t.a, (SELECT MAX('aa') FROM DUAL LIMIT 1));
+DROP FUNCTION f1;
diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h
index a59432058eb..41a51ee8d12 100644
--- a/sql/item_cmpfunc.h
+++ b/sql/item_cmpfunc.h
@@ -2117,16 +2117,7 @@ public:
   bool fix_length_and_dec();
   const char *func_name() const { return "regexp"; }
   enum precedence precedence() const { return CMP_PRECEDENCE; }
-  Item *get_copy(THD *thd, MEM_ROOT *mem_root)
-  { return get_item_copy<Item_func_regex>(thd, mem_root, this); }
-  Item *build_clone(THD *thd, MEM_ROOT *mem_root)
-  {
-    Item_func_regex *clone= (Item_func_regex*) Item_bool_func::build_clone(thd, mem_root);
-    if (clone)
-      clone->re.reset();
-    return clone;
-  }
-
+  Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return 0; }
   void print(String *str, enum_query_type query_type)
   {
     print_op(str, query_type);
@@ -2154,8 +2145,7 @@ public:
   bool fix_fields(THD *thd, Item **ref);
   bool fix_length_and_dec();
   const char *func_name() const { return "regexp_instr"; }
-  Item *get_copy(THD *thd, MEM_ROOT *mem_root)
-  { return get_item_copy<Item_func_regexp_instr>(thd, mem_root, this); }
+  Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return 0; }
 };
 
 
diff --git a/sql/item_func.h b/sql/item_func.h
index 0969cf2e3f0..4652f2c2d31 100644
--- a/sql/item_func.h
+++ b/sql/item_func.h
@@ -2439,15 +2439,7 @@ public:
   {
     return TRUE;
   }
-  Item *get_copy(THD *thd, MEM_ROOT *mem_root)
-  { return get_item_copy<Item_func_sp>(thd, mem_root, this); }
-  Item *build_clone(THD *thd, MEM_ROOT *mem_root)
-  {
-    Item_func_sp *clone= (Item_func_sp *) Item_func::build_clone(thd, mem_root);
-    if (clone)
-      clone->sp_result_field= NULL;
-    return clone;
-  }
+  Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return 0; }
   bool eval_not_null_tables(void *opt_arg)
   {
     not_null_tables_cache= 0;

From 255328d3931e8526c4248ec2af1f554911376c1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Thu, 26 Jul 2018 22:52:53 +0300
Subject: [PATCH 072/164] MDEV-16131 Assertion failed in
 dict_index_t::instant_field_value()

During a table-rebuilding online ALTER TABLE, if
dict_index_t::remove_instant() was invoked on the source table
(because it became empty), we would inadvertently change the way
how log records are written and parsed. We must keep the online_log
format unchanged throughout the whole table-rebuilding operation.

dict_col_t::def_t: Name the type of dict_col_t::def_val.

rec_get_n_add_field_len(), rec_set_n_add_field(): Define globally,
because these will be needed in row_log_table_low().

rec_init_offsets_temp(), rec_init_offsets_comp_ordinary(): Add
the parameter def_val for explicitly passing the default values
of the instantly added columns of the source table, so that
dict_index_t::instant_field_value() will not be called during
row_log_table_apply(). This allows us to consistently parse the
online_log records, even if the source table was converted
to the canonical non-instant format during the rebuild operation.

row_log_t::non_core_fields[]: The default values of the
instantly added columns on the source table; copied
during ha_innobase::prepare_inplace_alter_table()
while the table is exclusively locked.

row_log_t::instant_field_value(): Accessor to non_core_fields[],
analogous to dict_index_t::instant_field_value().

row_log_table_low(): Add fake_extra_size bytes to the record
header if the source table was converted to the canonical format
during the operation.

row_log_allocate(): Initialize row_log_t::non_core_fields.
---
 .../suite/innodb/r/instant_alter_debug.result | 36 +++++++++-
 .../suite/innodb/t/instant_alter_debug.test   | 44 +++++++++++-
 storage/innobase/include/dict0mem.h           |  2 +-
 storage/innobase/include/rem0rec.h            | 29 +++++++-
 storage/innobase/rem/rem0rec.cc               | 51 ++++++-------
 storage/innobase/row/row0log.cc               | 72 +++++++++++++++----
 6 files changed, 187 insertions(+), 47 deletions(-)

diff --git a/mysql-test/suite/innodb/r/instant_alter_debug.result b/mysql-test/suite/innodb/r/instant_alter_debug.result
index 389a04007e2..3c0a1600029 100644
--- a/mysql-test/suite/innodb/r/instant_alter_debug.result
+++ b/mysql-test/suite/innodb/r/instant_alter_debug.result
@@ -175,10 +175,42 @@ SET DEBUG_SYNC='now WAIT_FOR copied';
 BEGIN;
 INSERT INTO t1 SET b=1;
 ROLLBACK;
-disconnect stop_purge;
+connection stop_purge;
+COMMIT;
+connection default;
 InnoDB		2 transactions not purged
 SET DEBUG_SYNC='now SIGNAL logged';
-disconnect ddl;
+connection ddl;
+connection default;
 DROP TABLE t1;
 SET DEBUG_SYNC='RESET';
+#
+# MDEV-16131 Assertion failed in dict_index_t::instant_field_value()
+#
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t1 SET a=0;
+ALTER TABLE t1 ADD COLUMN b INT NOT NULL DEFAULT 2, ADD COLUMN c INT;
+connection stop_purge;
+START TRANSACTION WITH CONSISTENT SNAPSHOT;
+connection default;
+DELETE FROM t1;
+connection ddl;
+SET DEBUG_SYNC='row_log_table_apply1_before SIGNAL copied WAIT_FOR logged';
+ALTER TABLE t1 FORCE;
+disconnect stop_purge;
+connection default;
+SET DEBUG_SYNC = 'now WAIT_FOR copied';
+InnoDB		1 transactions not purged
+INSERT INTO t1 SET a=1;
+INSERT INTO t1 SET a=2,b=3,c=4;
+SET DEBUG_SYNC = 'now SIGNAL logged';
+connection ddl;
+disconnect ddl;
+connection default;
+SET DEBUG_SYNC = RESET;
+SELECT * FROM t1;
+a	b	c
+1	2	NULL
+2	3	4
+DROP TABLE t1;
 SET GLOBAL innodb_purge_rseg_truncate_frequency = @save_frequency;
diff --git a/mysql-test/suite/innodb/t/instant_alter_debug.test b/mysql-test/suite/innodb/t/instant_alter_debug.test
index 27056089767..5a73ed65d72 100644
--- a/mysql-test/suite/innodb/t/instant_alter_debug.test
+++ b/mysql-test/suite/innodb/t/instant_alter_debug.test
@@ -189,7 +189,9 @@ SET DEBUG_SYNC='now WAIT_FOR copied';
 BEGIN;
 INSERT INTO t1 SET b=1;
 ROLLBACK;
-disconnect stop_purge;
+connection stop_purge;
+COMMIT;
+connection default;
 
 # Wait for purge to empty the table.
 let $wait_all_purged=2;
@@ -197,8 +199,46 @@ let $wait_all_purged=2;
 let $wait_all_purged=0;
 
 SET DEBUG_SYNC='now SIGNAL logged';
-disconnect ddl;
+connection ddl;
+reap;
+connection default;
 DROP TABLE t1;
 SET DEBUG_SYNC='RESET';
 
+--echo #
+--echo # MDEV-16131 Assertion failed in dict_index_t::instant_field_value()
+--echo #
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t1 SET a=0;
+ALTER TABLE t1 ADD COLUMN b INT NOT NULL DEFAULT 2, ADD COLUMN c INT;
+
+connection stop_purge;
+START TRANSACTION WITH CONSISTENT SNAPSHOT;
+
+connection default;
+DELETE FROM t1;
+
+connection ddl;
+SET DEBUG_SYNC='row_log_table_apply1_before SIGNAL copied WAIT_FOR logged';
+send ALTER TABLE t1 FORCE;
+
+disconnect stop_purge;
+
+connection default;
+SET DEBUG_SYNC = 'now WAIT_FOR copied';
+let $wait_all_purged = 1;
+--source include/wait_all_purged.inc
+INSERT INTO t1 SET a=1;
+INSERT INTO t1 SET a=2,b=3,c=4;
+SET DEBUG_SYNC = 'now SIGNAL logged';
+
+connection ddl;
+reap;
+disconnect ddl;
+
+connection default;
+SET DEBUG_SYNC = RESET;
+SELECT * FROM t1;
+DROP TABLE t1;
+
 SET GLOBAL innodb_purge_rseg_truncate_frequency = @save_frequency;
diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h
index 6ed3b105da2..8d7c9d47e9f 100644
--- a/storage/innobase/include/dict0mem.h
+++ b/storage/innobase/include/dict0mem.h
@@ -593,7 +593,7 @@ struct dict_col_t{
 	inline void detach(const dict_index_t& index);
 
 	/** Data for instantly added columns */
-	struct {
+	struct def_t {
 		/** original default value of instantly added column */
 		const void*	data;
 		/** len of data, or UNIV_SQL_DEFAULT if unavailable */
diff --git a/storage/innobase/include/rem0rec.h b/storage/innobase/include/rem0rec.h
index cc8b92e99b1..3ee993944e9 100644
--- a/storage/innobase/include/rem0rec.h
+++ b/storage/innobase/include/rem0rec.h
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation.
+Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -312,6 +312,31 @@ rec_set_status(rec_t* rec, byte bits)
 		| bits;
 }
 
+/** Get the length of added field count in a REC_STATUS_COLUMNS_ADDED record.
+@param[in]	n_add_field	number of added fields, minus one
+@return	storage size of the field count, in bytes */
+inline unsigned rec_get_n_add_field_len(ulint n_add_field)
+{
+	ut_ad(n_add_field < REC_MAX_N_FIELDS);
+	return n_add_field < 0x80 ? 1 : 2;
+}
+
+/** Set the added field count in a REC_STATUS_COLUMNS_ADDED record.
+@param[in,out]	header	variable header of a REC_STATUS_COLUMNS_ADDED record
+@param[in]	n_add	number of added fields, minus 1
+@return	record header before the number of added fields */
+inline void rec_set_n_add_field(byte*& header, ulint n_add)
+{
+	ut_ad(n_add < REC_MAX_N_FIELDS);
+
+	if (n_add < 0x80) {
+		*header-- = byte(n_add);
+	} else {
+		*header-- = byte(n_add) | 0x80;
+		*header-- = byte(n_add >> 7);
+	}
+}
+
 /******************************************************//**
 The following function is used to retrieve the info and status
 bits of a record.  (Only compact records have status bits.)
@@ -962,6 +987,7 @@ rec_get_converted_size_temp(
 @param[in]	index	index of that the record belongs to
 @param[in,out]	offsets	offsets to the fields; in: rec_offs_n_fields(offsets)
 @param[in]	n_core	number of core fields (index->n_core_fields)
+@param[in]	def_val	default values for non-core fields
 @param[in]	status	REC_STATUS_ORDINARY or REC_STATUS_COLUMNS_ADDED */
 void
 rec_init_offsets_temp(
@@ -969,6 +995,7 @@ rec_init_offsets_temp(
 	const dict_index_t*	index,
 	ulint*			offsets,
 	ulint			n_core,
+	const dict_col_t::def_t*def_val,
 	rec_comp_status_t	status = REC_STATUS_ORDINARY)
 	MY_ATTRIBUTE((nonnull));
 /** Determine the offset to each field in temporary file.
diff --git a/storage/innobase/rem/rem0rec.cc b/storage/innobase/rem/rem0rec.cc
index 7d1a35d82e4..7a3aa44bc3f 100644
--- a/storage/innobase/rem/rem0rec.cc
+++ b/storage/innobase/rem/rem0rec.cc
@@ -237,15 +237,6 @@ rec_get_n_extern_new(
 	return(n_extern);
 }
 
-/** Get the length of added field count in a REC_STATUS_COLUMNS_ADDED record.
-@param[in]	n_add_field	number of added fields, minus one
-@return	storage size of the field count, in bytes */
-static inline unsigned rec_get_n_add_field_len(ulint n_add_field)
-{
-	ut_ad(n_add_field < REC_MAX_N_FIELDS);
-	return n_add_field < 0x80 ? 1 : 2;
-}
-
 /** Get the added field count in a REC_STATUS_COLUMNS_ADDED record.
 @param[in,out]	header	variable header of a REC_STATUS_COLUMNS_ADDED record
 @return	number of added fields */
@@ -264,22 +255,6 @@ static inline unsigned rec_get_n_add_field(const byte*& header)
 	return n_fields_add;
 }
 
-/** Set the added field count in a REC_STATUS_COLUMNS_ADDED record.
-@param[in,out]	header	variable header of a REC_STATUS_COLUMNS_ADDED record
-@param[in]	n_add	number of added fields, minus 1
-@return	record header before the number of added fields */
-static inline void rec_set_n_add_field(byte*& header, ulint n_add)
-{
-	ut_ad(n_add < REC_MAX_N_FIELDS);
-
-	if (n_add < 0x80) {
-		*header-- = byte(n_add);
-	} else {
-		*header-- = byte(n_add) | 0x80;
-		*header-- = byte(n_add >> 7);
-	}
-}
-
 /** Format of a leaf-page ROW_FORMAT!=REDUNDANT record */
 enum rec_leaf_format {
 	/** Temporary file record */
@@ -299,6 +274,8 @@ This is a special case of rec_init_offsets() and rec_get_offsets_func().
 @param[in]	rec	leaf-page record
 @param[in]	index	the index that the record belongs in
 @param[in]	n_core	number of core fields (index->n_core_fields)
+@param[in]	def_val	default values for non-core fields, or
+			NULL to refer to index->fields[].col->def_val
 @param[in,out]	offsets	offsets, with valid rec_offs_n_fields(offsets)
 @param[in]	format	record format */
 static inline
@@ -308,6 +285,7 @@ rec_init_offsets_comp_ordinary(
 	const dict_index_t*	index,
 	ulint*			offsets,
 	ulint			n_core,
+	const dict_col_t::def_t*def_val,
 	rec_leaf_format		format)
 {
 	ulint		offs		= 0;
@@ -379,7 +357,19 @@ ordinary:
 		ulint			len;
 
 		/* set default value flag */
-		if (i >= n_fields) {
+		if (i < n_fields) {
+		} else if (def_val) {
+			const dict_col_t::def_t& d = def_val[i - n_core];
+			if (!d.data) {
+				len = offs | REC_OFFS_SQL_NULL;
+				ut_ad(d.len == UNIV_SQL_NULL);
+			} else {
+				len = offs | REC_OFFS_DEFAULT;
+				any |= REC_OFFS_DEFAULT;
+			}
+
+			goto resolved;
+		} else {
 			ulint dlen;
 			if (!index->instant_field_value(i, &dlen)) {
 				len = offs | REC_OFFS_SQL_NULL;
@@ -618,12 +608,14 @@ rec_init_offsets(
 			ut_ad(leaf);
 			rec_init_offsets_comp_ordinary(rec, index, offsets,
 						       index->n_core_fields,
+						       NULL,
 						       REC_LEAF_COLUMNS_ADDED);
 			return;
 		case REC_STATUS_ORDINARY:
 			ut_ad(leaf);
 			rec_init_offsets_comp_ordinary(rec, index, offsets,
 						       index->n_core_fields,
+						       NULL,
 						       REC_LEAF_ORDINARY);
 			return;
 		}
@@ -1695,6 +1687,7 @@ rec_get_converted_size_temp(
 @param[in]	index	index of that the record belongs to
 @param[in,out]	offsets	offsets to the fields; in: rec_offs_n_fields(offsets)
 @param[in]	n_core	number of core fields (index->n_core_fields)
+@param[in]	def_val	default values for non-core fields
 @param[in]	status	REC_STATUS_ORDINARY or REC_STATUS_COLUMNS_ADDED */
 void
 rec_init_offsets_temp(
@@ -1702,6 +1695,7 @@ rec_init_offsets_temp(
 	const dict_index_t*	index,
 	ulint*			offsets,
 	ulint			n_core,
+	const dict_col_t::def_t*def_val,
 	rec_comp_status_t	status)
 {
 	ut_ad(status == REC_STATUS_ORDINARY
@@ -1710,7 +1704,7 @@ rec_init_offsets_temp(
 	if it was emptied during an ALTER TABLE operation. */
 	ut_ad(index->n_core_fields == n_core || !index->is_instant());
 	ut_ad(index->n_core_fields >= n_core);
-	rec_init_offsets_comp_ordinary(rec, index, offsets, n_core,
+	rec_init_offsets_comp_ordinary(rec, index, offsets, n_core, def_val,
 				       status == REC_STATUS_COLUMNS_ADDED
 				       ? REC_LEAF_TEMP_COLUMNS_ADDED
 				       : REC_LEAF_TEMP);
@@ -1729,7 +1723,8 @@ rec_init_offsets_temp(
 {
 	ut_ad(!index->is_instant());
 	rec_init_offsets_comp_ordinary(rec, index, offsets,
-				       index->n_core_fields, REC_LEAF_TEMP);
+				       index->n_core_fields, NULL,
+				       REC_LEAF_TEMP);
 }
 
 /** Convert a data tuple prefix to the temporary file format.
diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc
index db33910c6f2..386bd2786a0 100644
--- a/storage/innobase/row/row0log.cc
+++ b/storage/innobase/row/row0log.cc
@@ -227,6 +227,8 @@ struct row_log_t {
 	table could be emptied, so that table->is_instant() no longer holds,
 	but all log records must be in the "instant" format. */
 	unsigned	n_core_fields;
+	/** the default values of non-core fields when the operation started */
+	dict_col_t::def_t* non_core_fields;
 	bool		allow_not_null; /*!< Whether the alter ignore is being
 				used or if the sql mode is non-strict mode;
 				if not, NULL values will not be converted to
@@ -243,6 +245,14 @@ struct row_log_t {
 		ut_ad(n_core_fields <= index->n_fields);
 		return n_core_fields != index->n_fields;
 	}
+
+	const byte* instant_field_value(ulint n, ulint* len) const
+	{
+		ut_ad(n >= n_core_fields);
+		const dict_col_t::def_t& d= non_core_fields[n - n_core_fields];
+		*len = d.len;
+		return static_cast<const byte*>(d.data);
+	}
 };
 
 /** Create the file or online log if it does not exist.
@@ -935,15 +945,16 @@ row_log_table_low(
 	ulint			mrec_size;
 	ulint			avail_size;
 	const dict_index_t*	new_index;
+	row_log_t*		log = index->online_log;
 
-	new_index = dict_table_get_first_index(index->online_log->table);
+	new_index = dict_table_get_first_index(log->table);
 
 	ut_ad(dict_index_is_clust(index));
 	ut_ad(dict_index_is_clust(new_index));
 	ut_ad(!dict_index_is_online_ddl(new_index));
 	ut_ad(rec_offs_validate(rec, index, offsets));
 	ut_ad(rec_offs_n_fields(offsets) == dict_index_get_n_fields(index));
-	ut_ad(rec_offs_size(offsets) <= sizeof index->online_log->tail.buf);
+	ut_ad(rec_offs_size(offsets) <= sizeof log->tail.buf);
 	ut_ad(rw_lock_own_flagged(
 			&index->lock,
 			RW_LOCK_FLAG_S | RW_LOCK_FLAG_X | RW_LOCK_FLAG_SX));
@@ -970,7 +981,7 @@ row_log_table_low(
 
 	if (index->online_status != ONLINE_INDEX_CREATION
 	    || (index->type & DICT_CORRUPT) || index->table->corrupted
-	    || index->online_log->error != DB_SUCCESS) {
+	    || log->error != DB_SUCCESS) {
 		return;
 	}
 
@@ -987,14 +998,32 @@ row_log_table_low(
 	const ulint omit_size = REC_N_NEW_EXTRA_BYTES;
 
 	const ulint rec_extra_size = rec_offs_extra_size(offsets) - omit_size;
-	const bool is_instant = index->online_log->is_instant(index);
+	const bool is_instant = log->is_instant(index);
 	extra_size = rec_extra_size + is_instant;
 
+	unsigned fake_extra_size = 0;
+	byte fake_extra_buf[2];
+	if (is_instant && UNIV_UNLIKELY(!index->is_instant())) {
+		/* The source table was emptied after ALTER TABLE
+		started, and it was converted to non-instant format.
+		Because row_log_table_apply_op() expects to find
+		all records to be logged in the same way, we will
+		be unable to copy the rec_extra_size bytes from the
+		record header, but must convert them here. */
+		unsigned n_add = index->n_fields - 1 - log->n_core_fields;
+		fake_extra_size = rec_get_n_add_field_len(n_add);
+		ut_ad(fake_extra_size == 1 || fake_extra_size == 2);
+		extra_size += fake_extra_size;
+		byte* fake_extra = fake_extra_buf + fake_extra_size - 1;
+		rec_set_n_add_field(fake_extra, n_add);
+		ut_ad(fake_extra + 1 == fake_extra_buf);
+	}
+
 	mrec_size = ROW_LOG_HEADER_SIZE
 		+ (extra_size >= 0x80) + rec_offs_size(offsets) - omit_size
-		+ is_instant;
+		+ is_instant + fake_extra_size;
 
-	if (insert || index->online_log->same_pk) {
+	if (insert || log->same_pk) {
 		ut_ad(!old_pk);
 		old_pk_extra_size = old_pk_size = 0;
 	} else {
@@ -1012,8 +1041,7 @@ row_log_table_low(
 		mrec_size += 1/*old_pk_extra_size*/ + old_pk_size;
 	}
 
-	if (byte* b = row_log_table_open(index->online_log,
-					 mrec_size, &avail_size)) {
+	if (byte* b = row_log_table_open(log, mrec_size, &avail_size)) {
 		if (insert) {
 			*b++ = ROW_T_INSERT;
 		} else {
@@ -1038,20 +1066,23 @@ row_log_table_low(
 		}
 
 		if (is_instant) {
-			*b++ = rec_get_status(rec);
+			*b++ = fake_extra_size
+				? REC_STATUS_COLUMNS_ADDED
+				: rec_get_status(rec);
 		} else {
 			ut_ad(rec_get_status(rec) == REC_STATUS_ORDINARY);
 		}
 
 		memcpy(b, rec - rec_extra_size - omit_size, rec_extra_size);
 		b += rec_extra_size;
+		memcpy(b, fake_extra_buf, fake_extra_size);
+		b += fake_extra_size;
 		ulint len;
 		ulint trx_id_offs = rec_get_nth_field_offs(
 			offsets, index->n_uniq, &len);
 		ut_ad(len == DATA_TRX_ID_LEN);
 		memcpy(b, rec, rec_offs_data_size(offsets));
-		if (trx_read_trx_id(b + trx_id_offs)
-		    < index->online_log->min_trx) {
+		if (trx_read_trx_id(b + trx_id_offs) < log->min_trx) {
 			memcpy(b + trx_id_offs,
 			       reset_trx_id, sizeof reset_trx_id);
 		}
@@ -1576,7 +1607,7 @@ blob_done:
 		} else {
 			data = rec_get_nth_field(mrec, offsets, i, &len);
 			if (len == UNIV_SQL_DEFAULT) {
-				data = index->instant_field_value(i, &len);
+				data = log->instant_field_value(i, &len);
 			}
 			dfield_set_data(dfield, data, len);
 		}
@@ -2416,7 +2447,7 @@ row_log_table_apply_op(
 
 		rec_offs_set_n_fields(offsets, dup->index->n_fields);
 		rec_init_offsets_temp(mrec, dup->index, offsets,
-				      log->n_core_fields,
+				      log->n_core_fields, log->non_core_fields,
 				      is_instant
 				      ? static_cast<rec_comp_status_t>(
 					      *(mrec - extra_size))
@@ -2497,6 +2528,7 @@ row_log_table_apply_op(
 			rec_offs_set_n_fields(offsets, dup->index->n_fields);
 			rec_init_offsets_temp(mrec, dup->index, offsets,
 					      log->n_core_fields,
+					      log->non_core_fields,
 					      is_instant
 					      ? static_cast<rec_comp_status_t>(
 						      *(mrec - extra_size))
@@ -2598,6 +2630,7 @@ row_log_table_apply_op(
 			rec_offs_set_n_fields(offsets, dup->index->n_fields);
 			rec_init_offsets_temp(mrec, dup->index, offsets,
 					      log->n_core_fields,
+					      log->non_core_fields,
 					      is_instant
 					      ? static_cast<rec_comp_status_t>(
 						      *(mrec - extra_size))
@@ -3174,6 +3207,18 @@ row_log_allocate(
 	log->old_table = old_table;
 	log->n_rows = 0;
 
+	if (table && index->is_instant()) {
+		const unsigned n = log->n_core_fields;
+		log->non_core_fields = UT_NEW_ARRAY_NOKEY(
+			dict_col_t::def_t, index->n_fields - n);
+		for (unsigned i = n; i < index->n_fields; i++) {
+			log->non_core_fields[i - n]
+				= index->fields[i].col->def_val;
+		}
+	} else {
+		log->non_core_fields = NULL;
+	}
+
 	dict_index_set_online_status(index, ONLINE_INDEX_CREATION);
 	index->online_log = log;
 
@@ -3206,6 +3251,7 @@ row_log_free(
 	MONITOR_ATOMIC_DEC(MONITOR_ONLINE_CREATE_INDEX);
 
 	UT_DELETE(log->blobs);
+	UT_DELETE_ARRAY(log->non_core_fields);
 	row_log_block_free(log->tail);
 	row_log_block_free(log->head);
 	row_merge_file_destroy_low(log->fd);

From bd0b368119b48ffbb1e5ab3cd2887270c5c6840e Mon Sep 17 00:00:00 2001
From: Varun Gupta <varunraiko1803@gmail.com>
Date: Fri, 27 Jul 2018 11:34:34 +0530
Subject: [PATCH 073/164] Fix added along with a test for a case missed in the
 patch for MDEV-16751

---
 mysql-test/r/subselect_mat.result    | 17 +++++++++++++++++
 mysql-test/r/subselect_sj_mat.result | 17 +++++++++++++++++
 mysql-test/t/subselect_sj_mat.test   |  9 +++++++++
 sql/sql_select.cc                    |  8 ++++----
 4 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/mysql-test/r/subselect_mat.result b/mysql-test/r/subselect_mat.result
index 7f39f1fb788..eca3b760b65 100644
--- a/mysql-test/r/subselect_mat.result
+++ b/mysql-test/r/subselect_mat.result
@@ -2436,6 +2436,23 @@ a
 drop procedure prepare_data;
 set @@optimizer_switch= @save_optimizer_switch;
 drop table t1,t2,t3;
+CREATE TABLE t1 ( id int NOT NULL, key(id));
+INSERT INTO t1 VALUES (11),(12),(13),(14),(15),(16),(17),(18),(19);
+CREATE TABLE t2 (i1 int NOT NULL, i2 int NOT NULL);
+INSERT INTO t2 VALUES  (11,11),(12,12),(13,13);
+CREATE VIEW v1 AS SELECT t2.i1 FROM t2 where t2.i1 = t2.i2;
+explain SELECT 1 FROM t1 where t1.id IN (SELECT v1.i1 from v1);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	<subquery2>	ALL	distinct_key	NULL	NULL	NULL	3	
+1	PRIMARY	t1	ref	id	id	4	test.t2.i1	2	Using index
+2	MATERIALIZED	t2	ALL	NULL	NULL	NULL	NULL	3	Using where
+SELECT 1 FROM t1 where t1.id IN (SELECT v1.i1 from v1);
+1
+1
+1
+1
+drop table t1,t2;
+drop view v1;
 # End of 5.5 tests
 set @subselect_mat_test_optimizer_switch_value=null;
 set @@optimizer_switch='materialization=on,in_to_exists=off,semijoin=off';
diff --git a/mysql-test/r/subselect_sj_mat.result b/mysql-test/r/subselect_sj_mat.result
index 44006eadfc6..180c182a51a 100644
--- a/mysql-test/r/subselect_sj_mat.result
+++ b/mysql-test/r/subselect_sj_mat.result
@@ -2476,4 +2476,21 @@ a
 drop procedure prepare_data;
 set @@optimizer_switch= @save_optimizer_switch;
 drop table t1,t2,t3;
+CREATE TABLE t1 ( id int NOT NULL, key(id));
+INSERT INTO t1 VALUES (11),(12),(13),(14),(15),(16),(17),(18),(19);
+CREATE TABLE t2 (i1 int NOT NULL, i2 int NOT NULL);
+INSERT INTO t2 VALUES  (11,11),(12,12),(13,13);
+CREATE VIEW v1 AS SELECT t2.i1 FROM t2 where t2.i1 = t2.i2;
+explain SELECT 1 FROM t1 where t1.id IN (SELECT v1.i1 from v1);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	<subquery2>	ALL	distinct_key	NULL	NULL	NULL	3	
+1	PRIMARY	t1	ref	id	id	4	test.t2.i1	2	Using index
+2	MATERIALIZED	t2	ALL	NULL	NULL	NULL	NULL	3	Using where
+SELECT 1 FROM t1 where t1.id IN (SELECT v1.i1 from v1);
+1
+1
+1
+1
+drop table t1,t2;
+drop view v1;
 # End of 5.5 tests
diff --git a/mysql-test/t/subselect_sj_mat.test b/mysql-test/t/subselect_sj_mat.test
index 794545b53a7..c82c1e7acec 100644
--- a/mysql-test/t/subselect_sj_mat.test
+++ b/mysql-test/t/subselect_sj_mat.test
@@ -2221,4 +2221,13 @@ drop procedure prepare_data;
 set @@optimizer_switch= @save_optimizer_switch;
 drop table t1,t2,t3;
 
+CREATE TABLE t1 ( id int NOT NULL, key(id));
+INSERT INTO t1 VALUES (11),(12),(13),(14),(15),(16),(17),(18),(19);
+CREATE TABLE t2 (i1 int NOT NULL, i2 int NOT NULL);
+INSERT INTO t2 VALUES  (11,11),(12,12),(13,13);
+CREATE VIEW v1 AS SELECT t2.i1 FROM t2 where t2.i1 = t2.i2;
+explain SELECT 1 FROM t1 where t1.id IN (SELECT v1.i1 from v1);
+SELECT 1 FROM t1 where t1.id IN (SELECT v1.i1 from v1);
+drop table t1,t2;
+drop view v1;
 --echo # End of 5.5 tests
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 45a1372988b..b2b2bcde80c 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -7445,11 +7445,11 @@ bool JOIN_TAB::keyuse_is_valid_for_access_in_chosen_plan(JOIN *join,
   st_select_lex *sjm_sel= emb_sj_nest->sj_subq_pred->unit->first_select(); 
   for (uint i= 0; i < sjm_sel->item_list.elements; i++)
   {
-    DBUG_ASSERT(sjm_sel->ref_pointer_array[i]->type() == Item::FIELD_ITEM);
-    if (keyuse->val->type() == Item::FIELD_ITEM)
+    DBUG_ASSERT(sjm_sel->ref_pointer_array[i]->real_item()->type() == Item::FIELD_ITEM);
+    if (keyuse->val->real_item()->type() == Item::FIELD_ITEM)
     {
-      Field *field = ((Item_field*)sjm_sel->ref_pointer_array[i])->field;
-      if (field->eq(((Item_field*)keyuse->val)->field))
+      Field *field = ((Item_field*)sjm_sel->ref_pointer_array[i]->real_item())->field;
+      if (field->eq(((Item_field*)keyuse->val->real_item())->field))
         return true;
     }
   }

From 14306bcbec0b6a999985008f3d33602c08058006 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Fri, 27 Jul 2018 10:05:26 +0300
Subject: [PATCH 074/164] MDEV-16831: Galera test failure on
 galera_sst_mysqldump_with_key

Test case was not written correctly.
---
 .../r/galera_sst_mysqldump_with_key.result    | 95 ++++++++++++++++++-
 .../t/galera_sst_mysqldump_with_key.test      |  7 +-
 2 files changed, 97 insertions(+), 5 deletions(-)

diff --git a/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result b/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result
index 074481afc2d..bdf3844f2c5 100644
--- a/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result
+++ b/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result
@@ -9,7 +9,7 @@ CREATE USER sslsst;
 GRANT ALL PRIVILEGES ON *.* TO sslsst;
 GRANT USAGE ON *.* TO sslsst REQUIRE SSL;
 SET GLOBAL wsrep_sst_auth = 'sslsst:';
-Performing State Transfer on a server that has been temporarily disconnected
+Performing State Transfer on a server that has been shut down cleanly and restarted
 CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB;
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
@@ -27,8 +27,7 @@ INSERT INTO t1 VALUES ('node2_committed_before');
 INSERT INTO t1 VALUES ('node2_committed_before');
 INSERT INTO t1 VALUES ('node2_committed_before');
 COMMIT;
-Unloading wsrep provider ...
-SET GLOBAL wsrep_provider = 'none';
+Shutting down server ...
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
 INSERT INTO t1 VALUES ('node1_committed_during');
@@ -50,7 +49,95 @@ INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
-Loading wsrep provider ...
+Starting server ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+COMMIT;
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+COMMIT;
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+ROLLBACK;
+SELECT COUNT(*) = 35 FROM t1;
+COUNT(*) = 35
+1
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+SET AUTOCOMMIT=ON;
+SELECT COUNT(*) = 35 FROM t1;
+COUNT(*) = 35
+1
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+SET AUTOCOMMIT=ON;
+Performing State Transfer on a server that has been killed and restarted
+CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+COMMIT;
+Killing server ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+Performing --wsrep-recover ...
+Starting server ...
+Using --wsrep-start-position when starting mysqld ...
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
 INSERT INTO t1 VALUES ('node2_committed_after');
diff --git a/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.test b/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.test
index 0dbc63b531c..57244cb50c7 100644
--- a/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.test
+++ b/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.test
@@ -19,7 +19,12 @@ GRANT USAGE ON *.* TO sslsst REQUIRE SSL;
 
 SET GLOBAL wsrep_sst_auth = 'sslsst:';
 
---source suite/galera/include/galera_st_disconnect_slave.inc
+# We set the required mysqldump SST options here so that they are used every time the server is restarted during the test
+--let $start_mysqld_params = --wsrep_sst_auth=sst:'sslsst:' --wsrep_sst_method=mysqldump --wsrep-sst-receive-address=127.0.0.1:$NODE_MYPORT_2 --skip-grant-tables
+
+--source suite/galera/include/galera_st_shutdown_slave.inc
+--source suite/galera/include/galera_st_kill_slave.inc
+--source suite/galera/include/galera_st_kill_slave_ddl.inc
 
 --source include/auto_increment_offset_restore.inc
 --source suite/galera/include/galera_sst_restore.inc

From 0c745c743c65ceb6533babdf21556b633e81566c Mon Sep 17 00:00:00 2001
From: Michael Widenius <monty@mariadb.org>
Date: Sun, 29 Jul 2018 10:56:11 +0300
Subject: [PATCH 075/164] Don't give warnings from perror or resolveip with
 safemalloc

- Added my_end(0) before exit
- Fixed typo in mysql_install_db
---
 extra/perror.c              |  8 ++++++++
 extra/resolveip.c           | 17 +++++++++++++----
 scripts/mysql_install_db.sh |  2 +-
 3 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/extra/perror.c b/extra/perror.c
index 3bc5120db48..bd62401c8b2 100644
--- a/extra/perror.c
+++ b/extra/perror.c
@@ -105,11 +105,13 @@ get_one_option(int optid, const struct my_option *opt __attribute__((unused)),
     break;
   case 'V':
     print_version();
+    my_end(0);
     exit(0);
     break;
   case 'I':
   case '?':
     usage();
+    my_end(0);
     exit(0);
     break;
   }
@@ -122,7 +124,10 @@ static int get_options(int *argc,char ***argv)
   int ho_error;
 
   if ((ho_error=handle_options(argc, argv, my_long_options, get_one_option)))
+  {
+    my_end(0);
     exit(ho_error);
+  }
 
   if (!*argc && !print_all_codes)
   {
@@ -260,7 +265,10 @@ int main(int argc,char *argv[])
   MY_INIT(argv[0]);
 
   if (get_options(&argc,&argv))
+  {
+    my_end(0);
     exit(1);
+  }
 
   my_handler_error_register();
 
diff --git a/extra/resolveip.c b/extra/resolveip.c
index b6840d9d463..20941a05257 100644
--- a/extra/resolveip.c
+++ b/extra/resolveip.c
@@ -79,6 +79,7 @@ get_one_option(int optid, const struct my_option *opt __attribute__((unused)),
   case 'I':
   case '?':
     usage();
+    my_end(0);
     exit(0);
   }
   return 0;
@@ -91,7 +92,10 @@ static int get_options(int *argc,char ***argv)
   int ho_error;
 
   if ((ho_error=handle_options(argc, argv, my_long_options, get_one_option)))
+  {
+    my_end(0);
     exit(ho_error);
+  }
 
   if (*argc == 0)
   {
@@ -113,10 +117,14 @@ int main(int argc, char **argv)
   MY_INIT(argv[0]);
 
   if (get_options(&argc,&argv))
+  {
+    my_end(0);
     exit(1);
+  }
 
   while (argc--)
   {
+    my_bool do_more;
 #ifndef WIN32
     struct in_addr addr;
 #endif
@@ -125,13 +133,13 @@ int main(int argc, char **argv)
     /* Not compatible with IPv6!  Probably should use getnameinfo(). */
 #ifdef WIN32
     taddr = inet_addr(ip);
-    if(taddr != INADDR_NONE)
-    {
+    do_more= (taddr != INADDR_NONE);
 #else
-    if (inet_aton(ip, &addr) != 0)
-    {
+    if ((do_more= (inet_aton(ip, &addr) != 0)))
       taddr= addr.s_addr;
 #endif
+    if (do_more)
+    {
       if (taddr == htonl(INADDR_BROADCAST))
       {	
 	puts("Broadcast");
@@ -205,5 +213,6 @@ int main(int argc, char **argv)
       }
     }
   }
+  my_end(0);
   exit(error);
 }
diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh
index ad7c028a7af..de2ab3983b3 100644
--- a/scripts/mysql_install_db.sh
+++ b/scripts/mysql_install_db.sh
@@ -239,7 +239,7 @@ cannot_find_file()
   echo
   echo "If you compiled from source, you need to either run 'make install' to"
   echo "copy the software into the correct location ready for operation."
-  echo "If you don't want to do a full install, you can use the --srcddir"
+  echo "If you don't want to do a full install, you can use the --srcdir"
   echo "option to only install the mysql database and privilege tables"
   echo
   echo "If you are using a binary release, you must either be at the top"

From 340963351c5fb9dbeb03d6c887b18d328073ed79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Mon, 30 Jul 2018 10:39:42 +0300
Subject: [PATCH 076/164] Use a more precise argument for memset()

---
 storage/innobase/buf/buf0buddy.cc | 2 +-
 storage/xtradb/buf/buf0buddy.cc   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/storage/innobase/buf/buf0buddy.cc b/storage/innobase/buf/buf0buddy.cc
index 7a441b4239a..c8c39e0caaf 100644
--- a/storage/innobase/buf/buf0buddy.cc
+++ b/storage/innobase/buf/buf0buddy.cc
@@ -132,7 +132,7 @@ buf_buddy_stamp_free(
 	buf_buddy_free_t*	buf,	/*!< in/out: block to stamp */
 	ulint			i)	/*!< in: block size */
 {
-	ut_d(memset(buf, static_cast<int>(i), BUF_BUDDY_LOW << i));
+	ut_d(memset(&buf->stamp.bytes, int(i), BUF_BUDDY_LOW << i));
 	buf_buddy_mem_invalid(buf, i);
 	mach_write_to_4(buf->stamp.bytes + BUF_BUDDY_STAMP_OFFSET,
 			BUF_BUDDY_STAMP_FREE);
diff --git a/storage/xtradb/buf/buf0buddy.cc b/storage/xtradb/buf/buf0buddy.cc
index 1c50e71e687..ee687efc9ec 100644
--- a/storage/xtradb/buf/buf0buddy.cc
+++ b/storage/xtradb/buf/buf0buddy.cc
@@ -132,7 +132,7 @@ buf_buddy_stamp_free(
 	buf_buddy_free_t*	buf,	/*!< in/out: block to stamp */
 	ulint			i)	/*!< in: block size */
 {
-	ut_d(memset(buf, static_cast<int>(i), BUF_BUDDY_LOW << i));
+	ut_d(memset(&buf->stamp.bytes, int(i), BUF_BUDDY_LOW << i));
 	buf_buddy_mem_invalid(buf, i);
 	mach_write_to_4(buf->stamp.bytes + BUF_BUDDY_STAMP_OFFSET,
 			BUF_BUDDY_STAMP_FREE);

From 8bdd125067851a201348ae24cac7f9097d494d53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Mon, 30 Jul 2018 13:13:43 +0300
Subject: [PATCH 077/164] MDEV-16851 On schema mismatch in IMPORT TABLESPACE,
 display ROW_FORMAT in clear text

This is motivated by Oracle MySQL Bug #27542720 SCHEMA MISMATCH
- TABLE FLAGS DON'T MATCH, BUT FLAGS ARE NUMBERS
but using a different approach.

row_import::match_schema(): In case of a mismatch, display the
ROW_FORMAT and optionally KEY_BLOCK_SIZE of the .cfg file.
---
 .../suite/innodb/r/innodb-wl5522.result       | 263 +++++++++++++++++-
 mysql-test/suite/innodb/t/innodb-wl5522.test  | 258 ++++++++++++++++-
 storage/innobase/row/row0import.cc            |  59 +++-
 storage/xtradb/row/row0import.cc              |  52 +++-
 4 files changed, 608 insertions(+), 24 deletions(-)

diff --git a/mysql-test/suite/innodb/r/innodb-wl5522.result b/mysql-test/suite/innodb/r/innodb-wl5522.result
index 2116dfbf3fa..fb80580f797 100644
--- a/mysql-test/suite/innodb/r/innodb-wl5522.result
+++ b/mysql-test/suite/innodb/r/innodb-wl5522.result
@@ -1,4 +1,3 @@
-DROP TABLE IF EXISTS t1;
 SET GLOBAL innodb_file_per_table = 1;
 SELECT @@innodb_file_per_table;
 @@innodb_file_per_table
@@ -580,7 +579,7 @@ SELECT * FROM t1;
 ERROR HY000: Tablespace has been discarded for table 't1'
 restore: t1 .ibd and .cfg files
 ALTER TABLE t1 IMPORT TABLESPACE;
-ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x1 and the meta-data file has 0x0)
+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x1 and the meta-data file has 0x0; .cfg file uses ROW_FORMAT=REDUNDANT)
 unlink: t1.ibd
 unlink: t1.cfg
 DROP TABLE t1;
@@ -592,7 +591,19 @@ SELECT * FROM t1;
 ERROR HY000: Tablespace has been discarded for table 't1'
 restore: t1 .ibd and .cfg files
 ALTER TABLE t1 IMPORT TABLESPACE;
-ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x21 and the meta-data file has 0x0)
+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x21 and the meta-data file has 0x0; .cfg file uses ROW_FORMAT=REDUNDANT)
+unlink: t1.ibd
+unlink: t1.cfg
+DROP TABLE t1;
+CREATE TABLE t1(
+c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
+ALTER TABLE t1 DISCARD TABLESPACE;
+SELECT * FROM t1;
+ERROR HY000: Tablespace has been discarded for table 't1'
+restore: t1 .ibd and .cfg files
+ALTER TABLE t1 IMPORT TABLESPACE;
+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x29 and the meta-data file has 0x0; .cfg file uses ROW_FORMAT=REDUNDANT)
 unlink: t1.ibd
 unlink: t1.cfg
 DROP TABLE t1;
@@ -766,7 +777,7 @@ SELECT * FROM t1;
 ERROR HY000: Tablespace has been discarded for table 't1'
 restore: t1 .ibd and .cfg files
 ALTER TABLE t1 IMPORT TABLESPACE;
-ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x0 and the meta-data file has 0x1)
+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x0 and the meta-data file has 0x1; .cfg file uses ROW_FORMAT=COMPACT)
 unlink: t1.ibd
 unlink: t1.cfg
 DROP TABLE t1;
@@ -778,7 +789,19 @@ SELECT * FROM t1;
 ERROR HY000: Tablespace has been discarded for table 't1'
 restore: t1 .ibd and .cfg files
 ALTER TABLE t1 IMPORT TABLESPACE;
-ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x21 and the meta-data file has 0x1)
+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x21 and the meta-data file has 0x1; .cfg file uses ROW_FORMAT=COMPACT)
+unlink: t1.ibd
+unlink: t1.cfg
+DROP TABLE t1;
+CREATE TABLE t1(
+c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
+ALTER TABLE t1 DISCARD TABLESPACE;
+SELECT * FROM t1;
+ERROR HY000: Tablespace has been discarded for table 't1'
+restore: t1 .ibd and .cfg files
+ALTER TABLE t1 IMPORT TABLESPACE;
+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x29 and the meta-data file has 0x1; .cfg file uses ROW_FORMAT=COMPACT)
 unlink: t1.ibd
 unlink: t1.cfg
 DROP TABLE t1;
@@ -955,7 +978,7 @@ SELECT * FROM t1;
 ERROR HY000: Tablespace has been discarded for table 't1'
 restore: t1 .ibd and .cfg files
 ALTER TABLE t1 IMPORT TABLESPACE;
-ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x1 and the meta-data file has 0x21)
+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x1 and the meta-data file has 0x21; .cfg file uses ROW_FORMAT=DYNAMIC)
 unlink: t1.ibd
 unlink: t1.cfg
 DROP TABLE t1;
@@ -967,7 +990,19 @@ SELECT * FROM t1;
 ERROR HY000: Tablespace has been discarded for table 't1'
 restore: t1 .ibd and .cfg files
 ALTER TABLE t1 IMPORT TABLESPACE;
-ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x0 and the meta-data file has 0x21)
+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x0 and the meta-data file has 0x21; .cfg file uses ROW_FORMAT=DYNAMIC)
+unlink: t1.ibd
+unlink: t1.cfg
+DROP TABLE t1;
+CREATE TABLE t1(
+c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
+ALTER TABLE t1 DISCARD TABLESPACE;
+SELECT * FROM t1;
+ERROR HY000: Tablespace has been discarded for table 't1'
+restore: t1 .ibd and .cfg files
+ALTER TABLE t1 IMPORT TABLESPACE;
+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x29 and the meta-data file has 0x21; .cfg file uses ROW_FORMAT=DYNAMIC)
 unlink: t1.ibd
 unlink: t1.cfg
 DROP TABLE t1;
@@ -1026,6 +1061,220 @@ c1	c2
 42	1
 43	1
 DROP TABLE t1;
+CREATE TABLE t1(
+c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
+INSERT INTO t1(c2) VALUES(1);
+INSERT INTO t1(c2) SELECT c2 FROM t1;
+INSERT INTO t1(c2) SELECT c2 FROM t1;
+INSERT INTO t1(c2) SELECT c2 FROM t1;
+INSERT INTO t1(c2) SELECT c2 FROM t1;
+INSERT INTO t1(c2) SELECT c2 FROM t1;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` int(11) NOT NULL AUTO_INCREMENT,
+  `c2` int(11) DEFAULT NULL,
+  PRIMARY KEY (`c1`),
+  KEY `idx` (`c2`)
+) ENGINE=InnoDB AUTO_INCREMENT=59 DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED
+SELECT * FROM t1;
+c1	c2
+1	1
+2	1
+3	1
+4	1
+6	1
+7	1
+8	1
+9	1
+13	1
+14	1
+15	1
+16	1
+17	1
+18	1
+19	1
+20	1
+28	1
+29	1
+30	1
+31	1
+32	1
+33	1
+34	1
+35	1
+36	1
+37	1
+38	1
+39	1
+40	1
+41	1
+42	1
+43	1
+FLUSH TABLES t1 FOR EXPORT;
+backup: t1
+UNLOCK TABLES;
+DROP TABLE t1;
+CREATE TABLE t1(
+c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
+ALTER TABLE t1 DISCARD TABLESPACE;
+SELECT * FROM t1;
+ERROR HY000: Tablespace has been discarded for table 't1'
+restore: t1 .ibd and .cfg files
+ALTER TABLE t1 IMPORT TABLESPACE;
+CHECK TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+unlink: t1.cfg
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` int(11) NOT NULL AUTO_INCREMENT,
+  `c2` int(11) DEFAULT NULL,
+  PRIMARY KEY (`c1`),
+  KEY `idx` (`c2`)
+) ENGINE=InnoDB AUTO_INCREMENT=44 DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED
+SELECT * FROM t1;
+c1	c2
+1	1
+2	1
+3	1
+4	1
+6	1
+7	1
+8	1
+9	1
+13	1
+14	1
+15	1
+16	1
+17	1
+18	1
+19	1
+20	1
+28	1
+29	1
+30	1
+31	1
+32	1
+33	1
+34	1
+35	1
+36	1
+37	1
+38	1
+39	1
+40	1
+41	1
+42	1
+43	1
+DROP TABLE t1;
+CREATE TABLE t1(
+c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+ALTER TABLE t1 DISCARD TABLESPACE;
+SELECT * FROM t1;
+ERROR HY000: Tablespace has been discarded for table 't1'
+restore: t1 .ibd and .cfg files
+ALTER TABLE t1 IMPORT TABLESPACE;
+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x0 and the meta-data file has 0x29; .cfg file uses ROW_FORMAT=COMPRESSED)
+unlink: t1.ibd
+unlink: t1.cfg
+DROP TABLE t1;
+CREATE TABLE t1(
+c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=COMPACT;
+ALTER TABLE t1 DISCARD TABLESPACE;
+SELECT * FROM t1;
+ERROR HY000: Tablespace has been discarded for table 't1'
+restore: t1 .ibd and .cfg files
+ALTER TABLE t1 IMPORT TABLESPACE;
+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x1 and the meta-data file has 0x29; .cfg file uses ROW_FORMAT=COMPRESSED)
+unlink: t1.ibd
+unlink: t1.cfg
+DROP TABLE t1;
+CREATE TABLE t1(
+c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+ALTER TABLE t1 DISCARD TABLESPACE;
+SELECT * FROM t1;
+ERROR HY000: Tablespace has been discarded for table 't1'
+restore: t1 .ibd and .cfg files
+ALTER TABLE t1 IMPORT TABLESPACE;
+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x21 and the meta-data file has 0x29; .cfg file uses ROW_FORMAT=COMPRESSED)
+unlink: t1.ibd
+unlink: t1.cfg
+DROP TABLE t1;
+CREATE TABLE t1(
+c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+c2 INT, INDEX idx(c2)) ENGINE=InnoDB
+ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1;
+ALTER TABLE t1 DISCARD TABLESPACE;
+SELECT * FROM t1;
+ERROR HY000: Tablespace has been discarded for table 't1'
+restore: t1 .ibd and .cfg files
+ALTER TABLE t1 IMPORT TABLESPACE;
+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x23 and the meta-data file has 0x29; .cfg file uses ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8)
+unlink: t1.ibd
+unlink: t1.cfg
+DROP TABLE t1;
+CREATE TABLE t1(
+c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
+ALTER TABLE t1 DISCARD TABLESPACE;
+SELECT * FROM t1;
+ERROR HY000: Tablespace has been discarded for table 't1'
+restore: t1 .ibd and .cfg files
+ALTER TABLE t1 IMPORT TABLESPACE;
+CHECK TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+unlink: t1.cfg
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` int(11) NOT NULL AUTO_INCREMENT,
+  `c2` int(11) DEFAULT NULL,
+  PRIMARY KEY (`c1`),
+  KEY `idx` (`c2`)
+) ENGINE=InnoDB AUTO_INCREMENT=44 DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED
+SELECT * FROM t1;
+c1	c2
+1	1
+2	1
+3	1
+4	1
+6	1
+7	1
+8	1
+9	1
+13	1
+14	1
+15	1
+16	1
+17	1
+18	1
+19	1
+20	1
+28	1
+29	1
+30	1
+31	1
+32	1
+33	1
+34	1
+35	1
+36	1
+37	1
+38	1
+39	1
+40	1
+41	1
+42	1
+43	1
+DROP TABLE t1;
 call mtr.add_suppression("Got error -1 when reading table '.*'");
 call mtr.add_suppression("InnoDB: Error: tablespace id and flags in file '.*'.*");
 call mtr.add_suppression("InnoDB: The table .* doesn't have a corresponding tablespace, it was discarded");
diff --git a/mysql-test/suite/innodb/t/innodb-wl5522.test b/mysql-test/suite/innodb/t/innodb-wl5522.test
index c9e7748cb47..d50032c3be7 100644
--- a/mysql-test/suite/innodb/t/innodb-wl5522.test
+++ b/mysql-test/suite/innodb/t/innodb-wl5522.test
@@ -3,10 +3,6 @@
 
 -- source include/have_innodb.inc
 
---disable_warnings
-DROP TABLE IF EXISTS t1;
---enable_warnings
-
 let $innodb_file_per_table = `SELECT @@innodb_file_per_table`;
 let $innodb_file_format = `SELECT @@innodb_file_format`;
 
@@ -481,7 +477,7 @@ SELECT * FROM t1;
 DROP TABLE t1;
 
 #
-# Row format tests [EXPORT REDUNDANT - IMPORT COMPACT & DYNAMIC]
+# EXPORT ROW_FORMAT=REDUNDANT
 #
 CREATE TABLE t1(
 	c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
@@ -583,6 +579,29 @@ EOF
 
 DROP TABLE t1;
 
+CREATE TABLE t1(
+	c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+	c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
+ALTER TABLE t1 DISCARD TABLESPACE;
+--error ER_TABLESPACE_DISCARDED
+SELECT * FROM t1;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_discard_tablespaces("test", "t1");
+ib_restore_tablespaces("test", "t1");
+EOF
+
+--error ER_TABLE_SCHEMA_MISMATCH
+ALTER TABLE t1 IMPORT TABLESPACE;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_unlink_tablespace("test", "t1");
+EOF
+
+DROP TABLE t1;
+
 # This should be OK.
 CREATE TABLE t1(
 	c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
@@ -611,7 +630,7 @@ SELECT * FROM t1;
 DROP TABLE t1;
 
 #
-# Row format tests [EXPORT COMPACT - IMPORT REDUNDANT & DYNAMIC]
+# EXPORT ROW_FORMAT=COMPACT
 #
 CREATE TABLE t1(
 	c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
@@ -713,6 +732,29 @@ EOF
 
 DROP TABLE t1;
 
+CREATE TABLE t1(
+	c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+	c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
+ALTER TABLE t1 DISCARD TABLESPACE;
+--error ER_TABLESPACE_DISCARDED
+SELECT * FROM t1;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_discard_tablespaces("test", "t1");
+ib_restore_tablespaces("test", "t1");
+EOF
+
+--error ER_TABLE_SCHEMA_MISMATCH
+ALTER TABLE t1 IMPORT TABLESPACE;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_unlink_tablespace("test", "t1");
+EOF
+
+DROP TABLE t1;
+
 # This should be OK.
 CREATE TABLE t1(
 	c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
@@ -742,7 +784,7 @@ SELECT * FROM t1;
 DROP TABLE t1;
 
 #
-# Row format tests [EXPORT DYNAMIC- IMPORT REDUNDANT & DYNAMIC]
+# EXPORT ROW_FORMAT=DYNAMIC
 #
 CREATE TABLE t1(
 	c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
@@ -844,6 +886,29 @@ EOF
 
 DROP TABLE t1;
 
+CREATE TABLE t1(
+	c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+	c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
+ALTER TABLE t1 DISCARD TABLESPACE;
+--error ER_TABLESPACE_DISCARDED
+SELECT * FROM t1;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_discard_tablespaces("test", "t1");
+ib_restore_tablespaces("test", "t1");
+EOF
+
+--error ER_TABLE_SCHEMA_MISMATCH
+ALTER TABLE t1 IMPORT TABLESPACE;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_unlink_tablespace("test", "t1");
+EOF
+
+DROP TABLE t1;
+
 # This should be OK.
 CREATE TABLE t1(
 	c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
@@ -872,6 +937,185 @@ SELECT * FROM t1;
 
 DROP TABLE t1;
 
+#
+# EXPORT ROW_FORMAT=COMPRESSED
+#
+CREATE TABLE t1(
+	c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+	c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
+
+INSERT INTO t1(c2) VALUES(1);
+INSERT INTO t1(c2) SELECT c2 FROM t1;
+INSERT INTO t1(c2) SELECT c2 FROM t1;
+INSERT INTO t1(c2) SELECT c2 FROM t1;
+INSERT INTO t1(c2) SELECT c2 FROM t1;
+INSERT INTO t1(c2) SELECT c2 FROM t1;
+
+SHOW CREATE TABLE t1;
+SELECT * FROM t1;
+
+FLUSH TABLES t1 FOR EXPORT;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_backup_tablespaces("test", "t1");
+EOF
+
+UNLOCK TABLES;
+
+DROP TABLE t1;
+
+CREATE TABLE t1(
+	c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+	c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
+
+ALTER TABLE t1 DISCARD TABLESPACE;
+--error ER_TABLESPACE_DISCARDED
+SELECT * FROM t1;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_discard_tablespaces("test", "t1");
+ib_restore_tablespaces("test", "t1");
+EOF
+
+ALTER TABLE t1 IMPORT TABLESPACE;
+CHECK TABLE t1;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_cleanup("test", "t1");
+EOF
+
+SHOW CREATE TABLE t1;
+SELECT * FROM t1;
+
+DROP TABLE t1;
+
+CREATE TABLE t1(
+	c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+	c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+
+ALTER TABLE t1 DISCARD TABLESPACE;
+--error ER_TABLESPACE_DISCARDED
+SELECT * FROM t1;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_discard_tablespaces("test", "t1");
+ib_restore_tablespaces("test", "t1");
+EOF
+
+--error ER_TABLE_SCHEMA_MISMATCH
+ALTER TABLE t1 IMPORT TABLESPACE;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_unlink_tablespace("test", "t1");
+EOF
+
+DROP TABLE t1;
+
+CREATE TABLE t1(
+	c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+	c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=COMPACT;
+
+ALTER TABLE t1 DISCARD TABLESPACE;
+--error ER_TABLESPACE_DISCARDED
+SELECT * FROM t1;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_discard_tablespaces("test", "t1");
+ib_restore_tablespaces("test", "t1");
+EOF
+
+--error ER_TABLE_SCHEMA_MISMATCH
+ALTER TABLE t1 IMPORT TABLESPACE;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_unlink_tablespace("test", "t1");
+EOF
+
+DROP TABLE t1;
+
+CREATE TABLE t1(
+	c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+	c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+ALTER TABLE t1 DISCARD TABLESPACE;
+--error ER_TABLESPACE_DISCARDED
+SELECT * FROM t1;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_discard_tablespaces("test", "t1");
+ib_restore_tablespaces("test", "t1");
+EOF
+
+--error ER_TABLE_SCHEMA_MISMATCH
+ALTER TABLE t1 IMPORT TABLESPACE;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_unlink_tablespace("test", "t1");
+EOF
+
+DROP TABLE t1;
+
+
+CREATE TABLE t1(
+	c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+	c2 INT, INDEX idx(c2)) ENGINE=InnoDB
+	ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1;
+ALTER TABLE t1 DISCARD TABLESPACE;
+--error ER_TABLESPACE_DISCARDED
+SELECT * FROM t1;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_discard_tablespaces("test", "t1");
+ib_restore_tablespaces("test", "t1");
+EOF
+
+--error ER_TABLE_SCHEMA_MISMATCH
+ALTER TABLE t1 IMPORT TABLESPACE;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_unlink_tablespace("test", "t1");
+EOF
+
+DROP TABLE t1;
+
+# This should be OK.
+CREATE TABLE t1(
+	c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+	c2 INT, INDEX idx(c2)) ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
+
+ALTER TABLE t1 DISCARD TABLESPACE;
+--error ER_TABLESPACE_DISCARDED
+SELECT * FROM t1;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_discard_tablespaces("test", "t1");
+ib_restore_tablespaces("test", "t1");
+EOF
+
+ALTER TABLE t1 IMPORT TABLESPACE;
+CHECK TABLE t1;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_cleanup("test", "t1");
+EOF
+
+SHOW CREATE TABLE t1;
+SELECT * FROM t1;
+
+DROP TABLE t1;
+
 call mtr.add_suppression("Got error -1 when reading table '.*'");
 call mtr.add_suppression("InnoDB: Error: tablespace id and flags in file '.*'.*");
 call mtr.add_suppression("InnoDB: The table .* doesn't have a corresponding tablespace, it was discarded");
diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc
index f93d004d8f9..127f1df8df5 100644
--- a/storage/innobase/row/row0import.cc
+++ b/storage/innobase/row/row0import.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2012, 2018, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2015, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -1333,17 +1333,63 @@ row_import::match_schema(
 {
 	/* Do some simple checks. */
 
-	if ((m_table->flags ^ m_flags) & ~DICT_TF_MASK_DATA_DIR) {
+	if (ulint mismatch = (m_table->flags ^ m_flags)
+	    & ~DICT_TF_MASK_DATA_DIR) {
+		const char* msg;
+		if (mismatch & DICT_TF_MASK_ZIP_SSIZE) {
+			if ((m_table->flags & DICT_TF_MASK_ZIP_SSIZE)
+			    && (m_flags & DICT_TF_MASK_ZIP_SSIZE)) {
+				switch (m_flags & DICT_TF_MASK_ZIP_SSIZE) {
+				case 0U << DICT_TF_POS_ZIP_SSIZE:
+					goto uncompressed;
+				case 1U << DICT_TF_POS_ZIP_SSIZE:
+					msg = "ROW_FORMAT=COMPRESSED"
+						" KEY_BLOCK_SIZE=1";
+					break;
+				case 2U << DICT_TF_POS_ZIP_SSIZE:
+					msg = "ROW_FORMAT=COMPRESSED"
+						" KEY_BLOCK_SIZE=2";
+					break;
+				case 3U << DICT_TF_POS_ZIP_SSIZE:
+					msg = "ROW_FORMAT=COMPRESSED"
+						" KEY_BLOCK_SIZE=4";
+					break;
+				case 4U << DICT_TF_POS_ZIP_SSIZE:
+					msg = "ROW_FORMAT=COMPRESSED"
+						" KEY_BLOCK_SIZE=8";
+					break;
+				case 5U << DICT_TF_POS_ZIP_SSIZE:
+					msg = "ROW_FORMAT=COMPRESSED"
+						" KEY_BLOCK_SIZE=16";
+					break;
+				default:
+					msg = "strange KEY_BLOCK_SIZE";
+				}
+			} else if (m_flags & DICT_TF_MASK_ZIP_SSIZE) {
+				msg = "ROW_FORMAT=COMPRESSED";
+			} else {
+				goto uncompressed;
+			}
+		} else {
+uncompressed:
+			msg = (m_flags & DICT_TF_MASK_ATOMIC_BLOBS)
+				? "ROW_FORMAT=DYNAMIC"
+				: (m_flags & DICT_TF_MASK_COMPACT)
+				? "ROW_FORMAT=COMPACT"
+				: "ROW_FORMAT=REDUNDANT";
+		}
+
 		ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH,
 			"Table flags don't match, server table has 0x%x"
-			" and the meta-data file has 0x%lx",
-			m_table->flags, ulong(m_flags));
+			" and the meta-data file has 0x%lx;"
+			" .cfg file uses %s",
+			m_table->flags, ulong(m_flags), msg);
 
 		return(DB_ERROR);
 	} else if (m_table->n_cols != m_n_cols) {
 		ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH,
-			"Number of columns don't match, table has %u"
-			" columns but the tablespace meta-data file has "
+			"Number of columns don't match, table has %u "
+			"columns but the tablespace meta-data file has "
 			ULINTPF " columns",
 			m_table->n_cols, m_n_cols);
 
@@ -1822,7 +1868,6 @@ PageConverter::update_records(
 
 	while (!m_rec_iter.end()) {
 		rec_t*	rec = m_rec_iter.current();
-
 		ibool	deleted = rec_get_deleted_flag(rec, comp);
 
 		/* For the clustered index we have to adjust the BLOB
diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc
index 39e450bd7c5..127f1df8df5 100644
--- a/storage/xtradb/row/row0import.cc
+++ b/storage/xtradb/row/row0import.cc
@@ -1333,11 +1333,57 @@ row_import::match_schema(
 {
 	/* Do some simple checks. */
 
-	if ((m_table->flags ^ m_flags) & ~DICT_TF_MASK_DATA_DIR) {
+	if (ulint mismatch = (m_table->flags ^ m_flags)
+	    & ~DICT_TF_MASK_DATA_DIR) {
+		const char* msg;
+		if (mismatch & DICT_TF_MASK_ZIP_SSIZE) {
+			if ((m_table->flags & DICT_TF_MASK_ZIP_SSIZE)
+			    && (m_flags & DICT_TF_MASK_ZIP_SSIZE)) {
+				switch (m_flags & DICT_TF_MASK_ZIP_SSIZE) {
+				case 0U << DICT_TF_POS_ZIP_SSIZE:
+					goto uncompressed;
+				case 1U << DICT_TF_POS_ZIP_SSIZE:
+					msg = "ROW_FORMAT=COMPRESSED"
+						" KEY_BLOCK_SIZE=1";
+					break;
+				case 2U << DICT_TF_POS_ZIP_SSIZE:
+					msg = "ROW_FORMAT=COMPRESSED"
+						" KEY_BLOCK_SIZE=2";
+					break;
+				case 3U << DICT_TF_POS_ZIP_SSIZE:
+					msg = "ROW_FORMAT=COMPRESSED"
+						" KEY_BLOCK_SIZE=4";
+					break;
+				case 4U << DICT_TF_POS_ZIP_SSIZE:
+					msg = "ROW_FORMAT=COMPRESSED"
+						" KEY_BLOCK_SIZE=8";
+					break;
+				case 5U << DICT_TF_POS_ZIP_SSIZE:
+					msg = "ROW_FORMAT=COMPRESSED"
+						" KEY_BLOCK_SIZE=16";
+					break;
+				default:
+					msg = "strange KEY_BLOCK_SIZE";
+				}
+			} else if (m_flags & DICT_TF_MASK_ZIP_SSIZE) {
+				msg = "ROW_FORMAT=COMPRESSED";
+			} else {
+				goto uncompressed;
+			}
+		} else {
+uncompressed:
+			msg = (m_flags & DICT_TF_MASK_ATOMIC_BLOBS)
+				? "ROW_FORMAT=DYNAMIC"
+				: (m_flags & DICT_TF_MASK_COMPACT)
+				? "ROW_FORMAT=COMPACT"
+				: "ROW_FORMAT=REDUNDANT";
+		}
+
 		ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH,
 			"Table flags don't match, server table has 0x%x"
-			" and the meta-data file has 0x%lx",
-			m_table->flags, ulong(m_flags));
+			" and the meta-data file has 0x%lx;"
+			" .cfg file uses %s",
+			m_table->flags, ulong(m_flags), msg);
 
 		return(DB_ERROR);
 	} else if (m_table->n_cols != m_n_cols) {

From d17e9a02c4a08da9ac1b8f4c653f29f067d75a22 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Mon, 30 Jul 2018 14:05:24 +0300
Subject: [PATCH 078/164] Fix InnoDB/XtraDB warnings by GCC 8.2.0

---
 storage/innobase/buf/buf0dump.cc   |  2 +-
 storage/innobase/handler/i_s.cc    | 19 ++++++++-----------
 storage/innobase/row/row0import.cc | 26 ++++++++++++--------------
 storage/innobase/trx/trx0trx.cc    |  4 ++--
 storage/innobase/trx/trx0undo.cc   |  3 +--
 storage/xtradb/buf/buf0dump.cc     |  2 +-
 storage/xtradb/handler/i_s.cc      | 19 ++++++++-----------
 storage/xtradb/row/row0import.cc   | 26 ++++++++++++--------------
 storage/xtradb/trx/trx0trx.cc      |  4 ++--
 storage/xtradb/trx/trx0undo.cc     |  3 +--
 10 files changed, 48 insertions(+), 60 deletions(-)

diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc
index b2b8733c8ce..9642fc86018 100644
--- a/storage/innobase/buf/buf0dump.cc
+++ b/storage/innobase/buf/buf0dump.cc
@@ -203,7 +203,7 @@ buf_dump(
 #define SHOULD_QUIT()	(SHUTTING_DOWN() && obey_shutdown)
 
 	char	full_filename[OS_FILE_MAX_PATH];
-	char	tmp_filename[OS_FILE_MAX_PATH];
+	char	tmp_filename[OS_FILE_MAX_PATH + sizeof "incomplete"];
 	char	now[32];
 	FILE*	f;
 	ulint	i;
diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc
index f8aa3bd4637..de164e42273 100644
--- a/storage/innobase/handler/i_s.cc
+++ b/storage/innobase/handler/i_s.cc
@@ -1452,19 +1452,16 @@ i_s_cmp_fill_low(
 		clear it.  We could introduce mutex protection, but it
 		could cause a measureable performance hit in
 		page0zip.cc. */
-		table->field[1]->store(
-			static_cast<double>(zip_stat->compressed));
-		table->field[2]->store(
-			static_cast<double>(zip_stat->compressed_ok));
-		table->field[3]->store(
-			static_cast<double>(zip_stat->compressed_usec / 1000000));
-		table->field[4]->store(
-			static_cast<double>(zip_stat->decompressed));
-		table->field[5]->store(
-			static_cast<double>(zip_stat->decompressed_usec / 1000000));
+		table->field[1]->store(zip_stat->compressed, true);
+		table->field[2]->store(zip_stat->compressed_ok, true);
+		table->field[3]->store(zip_stat->compressed_usec / 1000000,
+				       true);
+		table->field[4]->store(zip_stat->decompressed, true);
+		table->field[5]->store(zip_stat->decompressed_usec / 1000000,
+				       true);
 
 		if (reset) {
-			memset(zip_stat, 0, sizeof *zip_stat);
+			new (zip_stat) page_zip_stat_t();
 		}
 
 		if (schema_table_store_record(thd, table)) {
diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc
index 127f1df8df5..dad55205bc2 100644
--- a/storage/innobase/row/row0import.cc
+++ b/storage/innobase/row/row0import.cc
@@ -105,18 +105,18 @@ struct row_index_t {
 struct row_import {
 	row_import() UNIV_NOTHROW
 		:
-		m_table(),
-		m_version(),
-		m_hostname(),
-		m_table_name(),
-		m_autoinc(),
-		m_page_size(),
-		m_flags(),
-		m_n_cols(),
-		m_cols(),
-		m_col_names(),
-		m_n_indexes(),
-		m_indexes(),
+		m_table(NULL),
+		m_version(0),
+		m_hostname(NULL),
+		m_table_name(NULL),
+		m_autoinc(0),
+		m_page_size(0),
+		m_flags(0),
+		m_n_cols(0),
+		m_cols(NULL),
+		m_col_names(NULL),
+		m_n_indexes(0),
+		m_indexes(NULL),
 		m_missing(true) { }
 
 	~row_import() UNIV_NOTHROW;
@@ -3558,8 +3558,6 @@ row_import_for_mysql(
 
 	row_import	cfg;
 
-	memset(&cfg, 0x0, sizeof(cfg));
-
 	err = row_import_read_cfg(table, trx->mysql_thd, cfg);
 
 	/* Check if the table column definitions match the contents
diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc
index 4a332d22cbc..2401783648a 100644
--- a/storage/innobase/trx/trx0trx.cc
+++ b/storage/innobase/trx/trx0trx.cc
@@ -2145,6 +2145,7 @@ trx_get_trx_by_xid_low(
 
 		if (trx->is_recovered
 		    && trx_state_eq(trx, TRX_STATE_PREPARED)
+		    && !trx->xid.is_null()
 		    && xid->gtrid_length == trx->xid.gtrid_length
 		    && xid->bqual_length == trx->xid.bqual_length
 		    && memcmp(xid->data, trx->xid.data,
@@ -2152,8 +2153,7 @@ trx_get_trx_by_xid_low(
 
 			/* Invalidate the XID, so that subsequent calls
 			will not find it. */
-			memset(&trx->xid, 0, sizeof(trx->xid));
-			trx->xid.formatID = -1;
+			trx->xid.null();
 			break;
 		}
 	}
diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc
index 370b3f181a1..b137c461ec3 100644
--- a/storage/innobase/trx/trx0undo.cc
+++ b/storage/innobase/trx/trx0undo.cc
@@ -1314,8 +1314,7 @@ trx_undo_mem_create_at_db_start(
 	/* Read X/Open XA transaction identification if it exists, or
 	set it to NULL. */
 
-	memset(&xid, 0, sizeof(xid));
-	xid.formatID = -1;
+	xid.null();
 
 	if (xid_exists == TRUE) {
 		trx_undo_read_xid(undo_header, &xid);
diff --git a/storage/xtradb/buf/buf0dump.cc b/storage/xtradb/buf/buf0dump.cc
index 51c41cc1b78..d5f07250f05 100644
--- a/storage/xtradb/buf/buf0dump.cc
+++ b/storage/xtradb/buf/buf0dump.cc
@@ -203,7 +203,7 @@ buf_dump(
 #define SHOULD_QUIT()	(SHUTTING_DOWN() && obey_shutdown)
 
 	char	full_filename[OS_FILE_MAX_PATH];
-	char	tmp_filename[OS_FILE_MAX_PATH];
+	char	tmp_filename[OS_FILE_MAX_PATH + sizeof "incomplete"];
 	char	now[32];
 	FILE*	f;
 	ulint	i;
diff --git a/storage/xtradb/handler/i_s.cc b/storage/xtradb/handler/i_s.cc
index 834bae399fb..b864830657e 100644
--- a/storage/xtradb/handler/i_s.cc
+++ b/storage/xtradb/handler/i_s.cc
@@ -1458,19 +1458,16 @@ i_s_cmp_fill_low(
 		clear it.  We could introduce mutex protection, but it
 		could cause a measureable performance hit in
 		page0zip.cc. */
-		table->field[1]->store(
-			static_cast<double>(zip_stat->compressed));
-		table->field[2]->store(
-			static_cast<double>(zip_stat->compressed_ok));
-		table->field[3]->store(
-			static_cast<double>(zip_stat->compressed_usec / 1000000));
-		table->field[4]->store(
-			static_cast<double>(zip_stat->decompressed));
-		table->field[5]->store(
-			static_cast<double>(zip_stat->decompressed_usec / 1000000));
+		table->field[1]->store(zip_stat->compressed, true);
+		table->field[2]->store(zip_stat->compressed_ok, true);
+		table->field[3]->store(zip_stat->compressed_usec / 1000000,
+				       true);
+		table->field[4]->store(zip_stat->decompressed, true);
+		table->field[5]->store(zip_stat->decompressed_usec / 1000000,
+				       true);
 
 		if (reset) {
-			memset(zip_stat, 0, sizeof *zip_stat);
+			new (zip_stat) page_zip_stat_t();
 		}
 
 		if (schema_table_store_record(thd, table)) {
diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc
index 127f1df8df5..dad55205bc2 100644
--- a/storage/xtradb/row/row0import.cc
+++ b/storage/xtradb/row/row0import.cc
@@ -105,18 +105,18 @@ struct row_index_t {
 struct row_import {
 	row_import() UNIV_NOTHROW
 		:
-		m_table(),
-		m_version(),
-		m_hostname(),
-		m_table_name(),
-		m_autoinc(),
-		m_page_size(),
-		m_flags(),
-		m_n_cols(),
-		m_cols(),
-		m_col_names(),
-		m_n_indexes(),
-		m_indexes(),
+		m_table(NULL),
+		m_version(0),
+		m_hostname(NULL),
+		m_table_name(NULL),
+		m_autoinc(0),
+		m_page_size(0),
+		m_flags(0),
+		m_n_cols(0),
+		m_cols(NULL),
+		m_col_names(NULL),
+		m_n_indexes(0),
+		m_indexes(NULL),
 		m_missing(true) { }
 
 	~row_import() UNIV_NOTHROW;
@@ -3558,8 +3558,6 @@ row_import_for_mysql(
 
 	row_import	cfg;
 
-	memset(&cfg, 0x0, sizeof(cfg));
-
 	err = row_import_read_cfg(table, trx->mysql_thd, cfg);
 
 	/* Check if the table column definitions match the contents
diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc
index 701ac3ed72d..8e62a7dba46 100644
--- a/storage/xtradb/trx/trx0trx.cc
+++ b/storage/xtradb/trx/trx0trx.cc
@@ -2421,6 +2421,7 @@ trx_get_trx_by_xid_low(
 
 		if (trx->is_recovered
 		    && trx_state_eq(trx, TRX_STATE_PREPARED)
+		    && !trx->xid.is_null()
 		    && xid->gtrid_length == trx->xid.gtrid_length
 		    && xid->bqual_length == trx->xid.bqual_length
 		    && memcmp(xid->data, trx->xid.data,
@@ -2428,8 +2429,7 @@ trx_get_trx_by_xid_low(
 
 			/* Invalidate the XID, so that subsequent calls
 			will not find it. */
-			memset(&trx->xid, 0, sizeof(trx->xid));
-			trx->xid.formatID = -1;
+			trx->xid.null();
 			break;
 		}
 	}
diff --git a/storage/xtradb/trx/trx0undo.cc b/storage/xtradb/trx/trx0undo.cc
index 370b3f181a1..b137c461ec3 100644
--- a/storage/xtradb/trx/trx0undo.cc
+++ b/storage/xtradb/trx/trx0undo.cc
@@ -1314,8 +1314,7 @@ trx_undo_mem_create_at_db_start(
 	/* Read X/Open XA transaction identification if it exists, or
 	set it to NULL. */
 
-	memset(&xid, 0, sizeof(xid));
-	xid.formatID = -1;
+	xid.null();
 
 	if (xid_exists == TRUE) {
 		trx_undo_read_xid(undo_header, &xid);

From fd378fc613851a12be346329d32e1666f10610d7 Mon Sep 17 00:00:00 2001
From: mleich1 <Matthias.Leich@mariadb.com>
Date: Mon, 30 Jul 2018 13:53:55 +0200
Subject: [PATCH 079/164] MDEV-16809 Allow full redo logging for ALTER TABLE
 Add the usual basic test for the variable innodb_log_optimize_ddl.

Signed-off-by: mleich1 <Matthias.Leich@mariadb.com>
---
 .../r/innodb_log_optimize_ddl_basic.result    | 39 +++++++++++
 .../t/innodb_log_optimize_ddl_basic.test      | 65 +++++++++++++++++++
 2 files changed, 104 insertions(+)
 create mode 100644 mysql-test/suite/sys_vars/r/innodb_log_optimize_ddl_basic.result
 create mode 100644 mysql-test/suite/sys_vars/t/innodb_log_optimize_ddl_basic.test

diff --git a/mysql-test/suite/sys_vars/r/innodb_log_optimize_ddl_basic.result b/mysql-test/suite/sys_vars/r/innodb_log_optimize_ddl_basic.result
new file mode 100644
index 00000000000..993bb13ebf0
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/innodb_log_optimize_ddl_basic.result
@@ -0,0 +1,39 @@
+SELECT COUNT(@@GLOBAL.innodb_log_optimize_ddl);
+COUNT(@@GLOBAL.innodb_log_optimize_ddl)
+1
+1 Expected
+SELECT COUNT(@@SESSION.innodb_log_optimize_ddl);
+ERROR HY000: Variable 'innodb_log_optimize_ddl' is a GLOBAL variable
+Expected error 'Variable is a GLOBAL variable'
+SELECT @@GLOBAL.innodb_log_optimize_ddl;
+@@GLOBAL.innodb_log_optimize_ddl
+1
+SELECT @@GLOBAL.innodb_log_optimize_ddl INTO @innodb_log_optimize_ddl_save;
+SET @@GLOBAL.innodb_log_optimize_ddl = ON;
+SET @@GLOBAL.innodb_log_optimize_ddl = OFF;
+SET @@GLOBAL.innodb_log_optimize_ddl = 13;
+ERROR 42000: Variable 'innodb_log_optimize_ddl' can't be set to the value of '13'
+SET @@GLOBAL.innodb_log_optimize_ddl = 'ABC';
+ERROR 42000: Variable 'innodb_log_optimize_ddl' can't be set to the value of 'ABC'
+SELECT @@GLOBAL.innodb_log_optimize_ddl = 0
+OR @@GLOBAL.innodb_log_optimize_ddl = 1 AS col;
+col
+1
+1 Expected
+SELECT @@innodb_log_optimize_ddl = @@GLOBAL.innodb_log_optimize_ddl AS col;
+col
+1
+1 Expected
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
+WHERE VARIABLE_NAME='innodb_log_optimize_ddl';
+VARIABLE_VALUE
+OFF
+SHOW VARIABLES WHERE VARIABLE_NAME='innodb_log_optimize_ddl';
+Variable_name	Value
+innodb_log_optimize_ddl	OFF
+SELECT @@local.innodb_log_optimize_ddl;
+ERROR HY000: Variable 'innodb_log_optimize_ddl' is a GLOBAL variable
+Expected error 'Variable is a GLOBAL variable'
+SELECT innodb_log_optimize_ddl;
+ERROR 42S22: Unknown column 'innodb_log_optimize_ddl' in 'field list'
+SET GLOBAL innodb_log_optimize_ddl = @innodb_log_optimize_ddl_save;
diff --git a/mysql-test/suite/sys_vars/t/innodb_log_optimize_ddl_basic.test b/mysql-test/suite/sys_vars/t/innodb_log_optimize_ddl_basic.test
new file mode 100644
index 00000000000..8e7612b0d9e
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/innodb_log_optimize_ddl_basic.test
@@ -0,0 +1,65 @@
+############# suite/sys_vars/t/innodb_log_optimize_ddl_basic.test #############
+#                                                                             #
+# Variable Name: innodb_log_optimize_ddl                                      #
+# Scope: Global                                                               #
+# Access Type: Dynamic                                                        #
+# Data Type: boolean                                                          #
+#                                                                             #
+# The variable was introduced by                                              #
+# MDEV-16809 Allow full redo logging for ALTER TABLE                          #
+#                                                                             #
+###############################################################################
+
+--source include/have_innodb.inc
+
+#### Reveal that the global innodb system variable exists
+SELECT COUNT(@@GLOBAL.innodb_log_optimize_ddl);
+--echo 1 Expected
+
+#### Reveal that no session innodb system variable exists
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT COUNT(@@SESSION.innodb_log_optimize_ddl);
+--echo Expected error 'Variable is a GLOBAL variable'
+
+#### Display the default value
+SELECT @@GLOBAL.innodb_log_optimize_ddl;
+
+SELECT @@GLOBAL.innodb_log_optimize_ddl INTO @innodb_log_optimize_ddl_save;
+#### Check if the value can be set
+SET @@GLOBAL.innodb_log_optimize_ddl = ON;
+SET @@GLOBAL.innodb_log_optimize_ddl = OFF;
+
+#### Check if disallowed values are refused
+--error ER_WRONG_VALUE_FOR_VAR
+SET @@GLOBAL.innodb_log_optimize_ddl = 13;
+--error ER_WRONG_VALUE_FOR_VAR
+SET @@GLOBAL.innodb_log_optimize_ddl = 'ABC';
+
+#### Check if the initial value was in the range of supported values
+# We use 0 and 1 in order to avoid a warning.
+SELECT @@GLOBAL.innodb_log_optimize_ddl = 0
+    OR @@GLOBAL.innodb_log_optimize_ddl = 1 AS col;
+--echo 1 Expected
+
+#### Check if the value presented without GLOBAL point is the same
+SELECT @@innodb_log_optimize_ddl = @@GLOBAL.innodb_log_optimize_ddl AS col;
+--echo 1 Expected
+
+#### Show the value presented in information_schema and SHOW VARIABLES
+# We do not want to get and than maybe suppress the print of
+#     Warning 1292 Truncated incorrect DOUBLE value: 'OFF'
+# and so we simply print the value and do not compare.
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
+WHERE VARIABLE_NAME='innodb_log_optimize_ddl';
+SHOW VARIABLES WHERE VARIABLE_NAME='innodb_log_optimize_ddl';
+
+#### Show that variants with @@local. and without @@ do not exist.
+--Error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT @@local.innodb_log_optimize_ddl;
+--echo Expected error 'Variable is a GLOBAL variable'
+--Error ER_BAD_FIELD_ERROR
+SELECT innodb_log_optimize_ddl;
+
+#### Restore the initial value
+SET GLOBAL innodb_log_optimize_ddl = @innodb_log_optimize_ddl_save;
+

From 29ddc6e9e37cbb74193b55e85a1b9cc3f47f28b6 Mon Sep 17 00:00:00 2001
From: Sachin Agarwal <sachin.z.agarwal@oracle.com>
Date: Thu, 17 May 2018 16:53:30 +0530
Subject: [PATCH 080/164] Bug #27326796 - MYSQL CRASH WITH INNODB ASSERTION
 FAILURE IN FILE PARS0PARS.CC

Problem:
As part of bug #24938374 fix, dict_operation_lock was not taken by
fts_optimize_thread while syncing fts cache.
Due to this change, alter query is able to update SYS_TABLE rows
simultaneously. Now when fts_optimizer_thread goes open index table,
It doesn't open index table if the record corresponding to that table is
set to REC_INFO_DELETED_FLAG in SYS_TABLES and hits an assert.

Fix:
If fts sync is already in progress, Alter query would wait for sync to
complete before renaming table.

RB: #19604
Reviewed by : Jimmy.Yang@oracle.com
---
 storage/innobase/fts/fts0fts.cc       | 57 +++++++++++----------------
 storage/innobase/handler/ha_innodb.cc | 37 ++++++++++++++++-
 storage/xtradb/fts/fts0fts.cc         | 57 +++++++++++----------------
 storage/xtradb/handler/ha_innodb.cc   | 37 ++++++++++++++++-
 4 files changed, 116 insertions(+), 72 deletions(-)

diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc
index 8ccc8afe7b4..86dd76bbf49 100644
--- a/storage/innobase/fts/fts0fts.cc
+++ b/storage/innobase/fts/fts0fts.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2011, 2017, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2011, 2018, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2016, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -869,37 +869,28 @@ fts_drop_index(
 
 			err = fts_drop_index_tables(trx, index);
 
-			for(;;) {
-				bool retry = false;
-				if (index->index_fts_syncing) {
-					retry = true;
-				}
-				if (!retry){
-					fts_free(table);
-					break;
-				}
+			while (index->index_fts_syncing
+				&& !trx_is_interrupted(trx)) {
 				DICT_BG_YIELD(trx);
 			}
+
+			fts_free(table);
+
 			return(err);
 		}
 
-		for(;;) {
-			bool retry = false;
-			if (index->index_fts_syncing) {
-				retry = true;
-			}
-			if (!retry){
-				current_doc_id = table->fts->cache->next_doc_id;
-				first_doc_id = table->fts->cache->first_doc_id;
-				fts_cache_clear(table->fts->cache);
-				fts_cache_destroy(table->fts->cache);
-				table->fts->cache = fts_cache_create(table);
-				table->fts->cache->next_doc_id = current_doc_id;
-				table->fts->cache->first_doc_id = first_doc_id;
-				break;
-			}
+		while (index->index_fts_syncing
+		       && !trx_is_interrupted(trx)) {
 			DICT_BG_YIELD(trx);
 		}
+
+		current_doc_id = table->fts->cache->next_doc_id;
+		first_doc_id = table->fts->cache->first_doc_id;
+		fts_cache_clear(table->fts->cache);
+		fts_cache_destroy(table->fts->cache);
+		table->fts->cache = fts_cache_create(table);
+		table->fts->cache->next_doc_id = current_doc_id;
+		table->fts->cache->first_doc_id = first_doc_id;
 	} else {
 		fts_cache_t*            cache = table->fts->cache;
 		fts_index_cache_t*      index_cache;
@@ -909,18 +900,14 @@ fts_drop_index(
 		index_cache = fts_find_index_cache(cache, index);
 
 		if (index_cache != NULL) {
-			for(;;) {
-				bool retry = false;
-				if (index->index_fts_syncing) {
-					retry = true;
-				}
-				if (!retry && index_cache->words) {
-					fts_words_free(index_cache->words);
-					rbt_free(index_cache->words);
-					break;
-				}
+			while (index->index_fts_syncing
+			       && !trx_is_interrupted(trx)) {
 				DICT_BG_YIELD(trx);
 			}
+			if (index_cache->words) {
+				fts_words_free(index_cache->words);
+				rbt_free(index_cache->words);
+			}
 
 			ib_vector_remove(cache->indexes, *(void**) index_cache);
 		}
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index b35eeb432fc..3959b49b600 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -10877,6 +10877,7 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result))
 dberr_t
 innobase_rename_table(
 /*==================*/
+	THD*            thd,    /*!< Connection thread handle */
 	trx_t*		trx,	/*!< in: transaction */
 	const char*	from,	/*!< in: old name of the table */
 	const char*	to)	/*!< in: new name of the table */
@@ -10902,6 +10903,36 @@ innobase_rename_table(
 
 	row_mysql_lock_data_dictionary(trx);
 
+	dict_table_t*   table = dict_table_open_on_name(norm_from, TRUE, FALSE,
+							DICT_ERR_IGNORE_NONE);
+
+	/* Since DICT_BG_YIELD has sleep for 250 milliseconds,
+	Convert lock_wait_timeout unit from second to 250 milliseconds */
+	long int lock_wait_timeout = thd_lock_wait_timeout(thd) * 4;
+	if (table != NULL) {
+		for (dict_index_t* index = dict_table_get_first_index(table);
+		     index != NULL;
+		     index = dict_table_get_next_index(index)) {
+
+			if (index->type & DICT_FTS) {
+				/* Found */
+				while (index->index_fts_syncing
+					&& !trx_is_interrupted(trx)
+					&& (lock_wait_timeout--) > 0) {
+					DICT_BG_YIELD(trx);
+				}
+			}
+		}
+		dict_table_close(table, TRUE, FALSE);
+	}
+
+	/* FTS sync is in progress. We shall timeout this operation */
+	if (lock_wait_timeout < 0) {
+		error = DB_LOCK_WAIT_TIMEOUT;
+		row_mysql_unlock_data_dictionary(trx);
+		DBUG_RETURN(error);
+	}
+
 	/* Transaction must be flagged as a locking transaction or it hasn't
 	been started yet. */
 
@@ -11011,7 +11042,7 @@ ha_innobase::rename_table(
 	++trx->will_lock;
 	trx_set_dict_operation(trx, TRX_DICT_OP_INDEX);
 
-	error = innobase_rename_table(trx, from, to);
+	error = innobase_rename_table(thd, trx, from, to);
 
 	DEBUG_SYNC(thd, "after_innobase_rename_table");
 
@@ -11055,6 +11086,10 @@ ha_innobase::rename_table(
 		my_error(ER_TABLE_EXISTS_ERROR, MYF(0), to);
 
 		error = DB_ERROR;
+	} else if (error == DB_LOCK_WAIT_TIMEOUT) {
+		my_error(ER_LOCK_WAIT_TIMEOUT, MYF(0), to);
+
+		error = DB_LOCK_WAIT;
 	}
 
 	DBUG_RETURN(convert_error_code_to_mysql(error, 0, NULL));
diff --git a/storage/xtradb/fts/fts0fts.cc b/storage/xtradb/fts/fts0fts.cc
index d2c20405e6a..36a1d234bf4 100644
--- a/storage/xtradb/fts/fts0fts.cc
+++ b/storage/xtradb/fts/fts0fts.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2011, 2017, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2011, 2018, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2016, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -869,37 +869,28 @@ fts_drop_index(
 
 			err = fts_drop_index_tables(trx, index);
 
-			for(;;) {
-				bool retry = false;
-				if (index->index_fts_syncing) {
-					retry = true;
-				}
-				if (!retry){
-					fts_free(table);
-					break;
-				}
+			while (index->index_fts_syncing
+				&& !trx_is_interrupted(trx)) {
 				DICT_BG_YIELD(trx);
 			}
+
+			fts_free(table);
+
 			return(err);
 		}
 
-		for(;;) {
-			bool retry = false;
-			if (index->index_fts_syncing) {
-				retry = true;
-			}
-			if (!retry){
-				current_doc_id = table->fts->cache->next_doc_id;
-				first_doc_id = table->fts->cache->first_doc_id;
-				fts_cache_clear(table->fts->cache);
-				fts_cache_destroy(table->fts->cache);
-				table->fts->cache = fts_cache_create(table);
-				table->fts->cache->next_doc_id = current_doc_id;
-				table->fts->cache->first_doc_id = first_doc_id;
-				break;
-			}
+		while (index->index_fts_syncing
+		       && !trx_is_interrupted(trx)) {
 			DICT_BG_YIELD(trx);
 		}
+
+		current_doc_id = table->fts->cache->next_doc_id;
+		first_doc_id = table->fts->cache->first_doc_id;
+		fts_cache_clear(table->fts->cache);
+		fts_cache_destroy(table->fts->cache);
+		table->fts->cache = fts_cache_create(table);
+		table->fts->cache->next_doc_id = current_doc_id;
+		table->fts->cache->first_doc_id = first_doc_id;
 	} else {
 		fts_cache_t*            cache = table->fts->cache;
 		fts_index_cache_t*      index_cache;
@@ -909,18 +900,14 @@ fts_drop_index(
 		index_cache = fts_find_index_cache(cache, index);
 
 		if (index_cache != NULL) {
-			for(;;) {
-				bool retry = false;
-				if (index->index_fts_syncing) {
-					retry = true;
-				}
-				if (!retry && index_cache->words) {
-					fts_words_free(index_cache->words);
-					rbt_free(index_cache->words);
-					break;
-				}
+			while (index->index_fts_syncing
+			       && !trx_is_interrupted(trx)) {
 				DICT_BG_YIELD(trx);
 			}
+			if (index_cache->words) {
+				fts_words_free(index_cache->words);
+				rbt_free(index_cache->words);
+			}
 
 			ib_vector_remove(cache->indexes, *(void**) index_cache);
 		}
diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc
index 89f4b9cb7b5..f153b6e28ae 100644
--- a/storage/xtradb/handler/ha_innodb.cc
+++ b/storage/xtradb/handler/ha_innodb.cc
@@ -11695,6 +11695,7 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result))
 dberr_t
 innobase_rename_table(
 /*==================*/
+	THD*            thd,    /*!< Connection thread handle */
 	trx_t*		trx,	/*!< in: transaction */
 	const char*	from,	/*!< in: old name of the table */
 	const char*	to)	/*!< in: new name of the table */
@@ -11720,6 +11721,36 @@ innobase_rename_table(
 
 	row_mysql_lock_data_dictionary(trx);
 
+	dict_table_t*   table = dict_table_open_on_name(norm_from, TRUE, FALSE,
+							DICT_ERR_IGNORE_NONE);
+
+	/* Since DICT_BG_YIELD has sleep for 250 milliseconds,
+	Convert lock_wait_timeout unit from second to 250 milliseconds */
+	long int lock_wait_timeout = thd_lock_wait_timeout(thd) * 4;
+	if (table != NULL) {
+		for (dict_index_t* index = dict_table_get_first_index(table);
+		     index != NULL;
+		     index = dict_table_get_next_index(index)) {
+
+			if (index->type & DICT_FTS) {
+				/* Found */
+				while (index->index_fts_syncing
+					&& !trx_is_interrupted(trx)
+					&& (lock_wait_timeout--) > 0) {
+					DICT_BG_YIELD(trx);
+				}
+			}
+		}
+		dict_table_close(table, TRUE, FALSE);
+	}
+
+	/* FTS sync is in progress. We shall timeout this operation */
+	if (lock_wait_timeout < 0) {
+		error = DB_LOCK_WAIT_TIMEOUT;
+		row_mysql_unlock_data_dictionary(trx);
+		DBUG_RETURN(error);
+	}
+
 	/* Transaction must be flagged as a locking transaction or it hasn't
 	been started yet. */
 
@@ -11834,7 +11865,7 @@ ha_innobase::rename_table(
 	++trx->will_lock;
 	trx_set_dict_operation(trx, TRX_DICT_OP_INDEX);
 
-	error = innobase_rename_table(trx, from, to);
+	error = innobase_rename_table(thd, trx, from, to);
 
 	DEBUG_SYNC(thd, "after_innobase_rename_table");
 
@@ -11878,6 +11909,10 @@ ha_innobase::rename_table(
 		my_error(ER_TABLE_EXISTS_ERROR, MYF(0), to);
 
 		error = DB_ERROR;
+	} else if (error == DB_LOCK_WAIT_TIMEOUT) {
+		my_error(ER_LOCK_WAIT_TIMEOUT, MYF(0), to);
+
+		error = DB_LOCK_WAIT;
 	}
 
 	DBUG_RETURN(convert_error_code_to_mysql(error, 0, NULL));

From a49ec980422e04ce1a37344be09aa6254f16fa2a Mon Sep 17 00:00:00 2001
From: Karthik Kamath <karthik.kamath@oracle.com>
Date: Wed, 14 Feb 2018 09:35:18 +0530
Subject: [PATCH 081/164]

---
 storage/xtradb/handler/ha_innodb.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc
index 94e49d4897a..1fe93f0ff9d 100644
--- a/storage/xtradb/handler/ha_innodb.cc
+++ b/storage/xtradb/handler/ha_innodb.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2000, 2015, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2000, 2018, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2008, 2009 Google Inc.
 Copyright (c) 2009, Percona Inc.
 
@@ -10689,8 +10689,10 @@ ha_innobase::start_stmt(
 		case SQLCOM_INSERT:
 		case SQLCOM_UPDATE:
 		case SQLCOM_DELETE:
+		case SQLCOM_REPLACE:
 			init_table_handle_for_HANDLER();
 			prebuilt->select_lock_type = LOCK_X;
+			prebuilt->stored_select_lock_type = LOCK_X;
 			error = row_lock_table_for_mysql(prebuilt, NULL, 1);
 
 			if (error != DB_SUCCESS) {

From 5ed2da9587c0dd6dcf9cfcb19f5e434488d79e32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Mon, 30 Jul 2018 16:28:20 +0300
Subject: [PATCH 082/164] Apply the 5.6.40 security fixes to XtraDB

We did not merge Percona XtraDB 5.6.40-84.0 yet.
The changes in it are mostly cosmetic, except for
2 bug fixes from Oracle MySQL 5.6.40, which could
be security bugs.

This was achieved by taking the applicable parts
of an earlier InnoDB commit to XtraDB:

git diff 15ec8c2f28f08517ecbffb959d756b4bdd53ab45{~,} storage/innobase|
sed -e s+/innobase/+/xtradb/+|patch -p1
---
 storage/xtradb/handler/handler0alter.cc | 51 ++++++++++++++++---------
 storage/xtradb/row/row0log.cc           | 12 +++++-
 2 files changed, 43 insertions(+), 20 deletions(-)

diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc
index 755c07848e1..151c3237b2b 100644
--- a/storage/xtradb/handler/handler0alter.cc
+++ b/storage/xtradb/handler/handler0alter.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2005, 2017, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2005, 2018, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -4782,13 +4782,15 @@ processed_field:
 }
 
 /** Get the auto-increment value of the table on commit.
-@param ha_alter_info	Data used during in-place alter
-@param ctx		In-place ALTER TABLE context
-@param altered_table	MySQL table that is being altered
-@param old_table	MySQL table as it is before the ALTER operation
-@return the next auto-increment value (0 if not present) */
+@param[in] ha_alter_info	Data used during in-place alter
+@param[in,out] ctx		In-place ALTER TABLE context
+				return autoinc value in ctx->max_autoinc
+@param altered_table[in]	MySQL table that is being altered
+@param old_table[in]	MySQL table as it is before the ALTER operation
+retval true Failure
+@retval false Success*/
 static MY_ATTRIBUTE((nonnull, warn_unused_result))
-ulonglong
+bool
 commit_get_autoinc(
 /*===============*/
 	Alter_inplace_info*	ha_alter_info,
@@ -4796,23 +4798,28 @@ commit_get_autoinc(
 	const TABLE*		altered_table,
 	const TABLE*		old_table)
 {
-	ulonglong		max_autoinc;
 
 	DBUG_ENTER("commit_get_autoinc");
 
 	if (!altered_table->found_next_number_field) {
 		/* There is no AUTO_INCREMENT column in the table
 		after the ALTER operation. */
-		max_autoinc = 0;
+		ctx->max_autoinc = 0;
 	} else if (ctx->add_autoinc != ULINT_UNDEFINED) {
 		/* An AUTO_INCREMENT column was added. Get the last
 		value from the sequence, which may be based on a
 		supplied AUTO_INCREMENT value. */
-		max_autoinc = ctx->sequence.last();
+		ctx->max_autoinc = ctx->sequence.last();
 	} else if ((ha_alter_info->handler_flags
 		    & Alter_inplace_info::CHANGE_CREATE_OPTION)
 		   && (ha_alter_info->create_info->used_fields
 		       & HA_CREATE_USED_AUTO)) {
+
+		/* Check if the table is discarded */
+		if(dict_table_is_discarded(ctx->old_table)) {
+			DBUG_RETURN(true);
+		}
+
 		/* An AUTO_INCREMENT value was supplied, but the table was not
 		rebuilt. Get the user-supplied value or the last value from the
 		sequence. */
@@ -4827,7 +4834,8 @@ commit_get_autoinc(
 		dict_index_t*	index = dict_table_get_index_on_name(
 			ctx->old_table, autoinc_key->name);
 
-		max_autoinc = ha_alter_info->create_info->auto_increment_value;
+		ctx->max_autoinc =
+			ha_alter_info->create_info->auto_increment_value;
 
 		dict_table_autoinc_lock(ctx->old_table);
 
@@ -4836,8 +4844,8 @@ commit_get_autoinc(
 
 		if (err != DB_SUCCESS) {
 			ut_ad(0);
-			max_autoinc = 0;
-		} else if (max_autoinc <= max_value_table) {
+			ctx->max_autoinc = 0;
+		} else if (ctx->max_autoinc <= max_value_table) {
 			ulonglong	col_max_value;
 			ulonglong	offset;
 
@@ -4845,7 +4853,7 @@ commit_get_autoinc(
 				old_table->found_next_number_field);
 
 			offset = ctx->prebuilt->autoinc_offset;
-			max_autoinc = innobase_next_autoinc(
+			ctx->max_autoinc = innobase_next_autoinc(
 				max_value_table, 1, 1, offset,
 				col_max_value);
 		}
@@ -4855,11 +4863,11 @@ commit_get_autoinc(
 		Read the old counter value from the table. */
 		ut_ad(old_table->found_next_number_field);
 		dict_table_autoinc_lock(ctx->old_table);
-		max_autoinc = ctx->old_table->autoinc;
+		ctx->max_autoinc = ctx->old_table->autoinc;
 		dict_table_autoinc_unlock(ctx->old_table);
 	}
 
-	DBUG_RETURN(max_autoinc);
+	DBUG_RETURN(false);
 }
 
 /** Add or drop foreign key constraints to the data dictionary tables,
@@ -5828,8 +5836,13 @@ ha_innobase::commit_inplace_alter_table(
 
 		DBUG_ASSERT(new_clustered == ctx->need_rebuild());
 
-		ctx->max_autoinc = commit_get_autoinc(
-			ha_alter_info, ctx, altered_table, table);
+		if (commit_get_autoinc(ha_alter_info, ctx, altered_table,
+			table)) {
+			fail = true;
+			my_error(ER_TABLESPACE_DISCARDED, MYF(0),
+				 table->s->table_name.str);
+			goto rollback_trx;
+		}
 
 		if (ctx->need_rebuild()) {
 			ctx->tmp_name = dict_mem_create_temporary_tablename(
@@ -5861,6 +5874,8 @@ ha_innobase::commit_inplace_alter_table(
 #endif
 	}
 
+rollback_trx:
+
 	/* Commit or roll back the changes to the data dictionary. */
 
 	if (fail) {
diff --git a/storage/xtradb/row/row0log.cc b/storage/xtradb/row/row0log.cc
index c239722e4f7..7cab4cbbebb 100644
--- a/storage/xtradb/row/row0log.cc
+++ b/storage/xtradb/row/row0log.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2011, 2018, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -2631,7 +2631,15 @@ all_done:
 
 	while (!trx_is_interrupted(trx)) {
 		mrec = next_mrec;
-		ut_ad(mrec < mrec_end);
+		ut_ad(mrec <= mrec_end);
+
+		if (mrec == mrec_end) {
+			/* We are at the end of the log.
+			   Mark the replay all_done. */
+			if (has_index_lock) {
+				goto all_done;
+			}
+		}
 
 		if (!has_index_lock) {
 			/* We are applying operations from a different

From e52315a4a2c7e752e786f9fcf63b0b5685b0d474 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Mon, 30 Jul 2018 18:06:30 +0300
Subject: [PATCH 083/164] MDEV-16855 Fix fts_sync_synchronization in InnoDB
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is a backport of the following fix from MySQL 5.7.23.
Some code refactoring has been omitted, and the test case has
been adapted to MariaDB.

commit 7a689acaa65e9d602575f7aa53fe36a64a07460f
Author: Krzysztof Kapuścik <krzysztof.kapuscik@oracle.com>
Date:   Tue Mar 13 12:34:03 2018 +0100

Bug#27082268 Invalid FTS sync synchronization

The fix closes two issues:
Bug #27082268 - INNODB: FAILING ASSERTION: SYM_NODE->TABLE != NULL DURING FTS SYNC
Bug #27095935 - DEADLOCK BETWEEN FTS_DROP_INDEX AND FTS_OPTIMIZE_SYNC_TABLE

Both issues were related to a FTS cache sync being done during
operations that perfomed DDL actions on internal FTS tables
(ALTER TABLE, TRUNCATE). In some cases the FTS tables and/or
internal cache structures could get removed while still being
used to perform FTS synchronization leading to crashes. In other
the sync operations could not get finishes as it was waiting for
dict lock which was taken by thread waiting for the background
sync to be finished.

The changes done includes:
- Stopping background operations during ALTER TABLE and TRUNCATE.
- Removal of unused code in FTS.
- Cleanup of FTS sync related code to make it more readable and
easier to maintain.

RB#18262
---
 mysql-test/suite/innodb_fts/r/sync_ddl.result | 117 ++++++++++++
 mysql-test/suite/innodb_fts/t/sync_ddl.test   | 177 ++++++++++++++++++
 storage/innobase/fts/fts0fts.cc               |  19 +-
 storage/innobase/fts/fts0opt.cc               | 175 ++---------------
 storage/innobase/handler/handler0alter.cc     |  66 +++----
 storage/innobase/include/fts0fts.h            |   8 +-
 storage/innobase/include/fts0priv.h           |  18 +-
 storage/innobase/row/row0mysql.cc             |  12 ++
 storage/xtradb/fts/fts0fts.cc                 |  19 +-
 storage/xtradb/fts/fts0opt.cc                 | 175 ++---------------
 storage/xtradb/handler/handler0alter.cc       |  66 +++----
 storage/xtradb/include/fts0fts.h              |   8 +-
 storage/xtradb/include/fts0priv.h             |  18 +-
 storage/xtradb/row/row0mysql.cc               |  12 ++
 14 files changed, 450 insertions(+), 440 deletions(-)
 create mode 100644 mysql-test/suite/innodb_fts/r/sync_ddl.result
 create mode 100644 mysql-test/suite/innodb_fts/t/sync_ddl.test

diff --git a/mysql-test/suite/innodb_fts/r/sync_ddl.result b/mysql-test/suite/innodb_fts/r/sync_ddl.result
new file mode 100644
index 00000000000..1e98594b28e
--- /dev/null
+++ b/mysql-test/suite/innodb_fts/r/sync_ddl.result
@@ -0,0 +1,117 @@
+CREATE TABLE t1 (
+id INT AUTO_INCREMENT NOT NULL PRIMARY KEY,
+value VARCHAR(1024)
+) ENGINE=InnoDB;
+CREATE FULLTEXT INDEX idx1 ON t1(value);
+Warnings:
+Warning	124	InnoDB rebuilding table to add column FTS_DOC_ID
+SET @save_debug = @@GLOBAL.debug_dbug;
+SET GLOBAL debug_dbug = '+d,fts_instrument_sync_request,fts_instrument_sync_before_syncing,ib_trunc_sleep_before_fts_cache_clear';
+INSERT INTO t1 (value) VALUES
+('By default or with the IN NATURAL LANGUAGE MODE modifier')
+;
+TRUNCATE TABLE t1;
+DROP TABLE t1;
+SET GLOBAL debug_dbug = @save_debug;
+CREATE TABLE t1 (
+id INT AUTO_INCREMENT NOT NULL PRIMARY KEY,
+value VARCHAR(1024)
+) ENGINE=InnoDB;
+CREATE FULLTEXT INDEX idx1 ON t1(value);
+Warnings:
+Warning	124	InnoDB rebuilding table to add column FTS_DOC_ID
+SET GLOBAL debug_dbug = '+d,fts_instrument_sync_request,fts_instrument_write_words_before_select_index,ib_trunc_sleep_before_fts_cache_clear';
+INSERT INTO t1 (value) VALUES
+('By default or with the IN NATURAL LANGUAGE MODE modifier'),
+('performs a natural language search for a string'),
+('collection is a set of one or more columns included'),
+('returns a relevance value; that is, a similarity measure'),
+('and the text in that row in the columns named in'),
+('By default, the search is performed in case-insensitive'),
+('sensitive full-text search, use a binary collation '),
+('example, a column that uses the latin1 character'),
+('collation of latin1_bin to make it case sensitive')
+;
+TRUNCATE TABLE t1;
+DROP TABLE t1;
+SET GLOBAL debug_dbug = @save_debug;
+CREATE TABLE t1 (
+value VARCHAR(1024)
+) ENGINE=InnoDB;
+CREATE FULLTEXT INDEX idx1 ON t1(value);
+Warnings:
+Warning	124	InnoDB rebuilding table to add column FTS_DOC_ID
+SET GLOBAL debug_dbug = '+d,fts_instrument_sync_request,fts_instrument_msg_sync_sleep';
+INSERT INTO t1 (value) VALUES
+('By default or with the IN NATURAL LANGUAGE MODE modifier'),
+('performs a natural language search for a string'),
+('collection is a set of one or more columns included'),
+('returns a relevance value; that is, a similarity measure'),
+('and the text in that row in the columns named in'),
+('By default, the search is performed in case-insensitive'),
+('sensitive full-text search, use a binary collation '),
+('example, a column that uses the latin1 character'),
+('collation of latin1_bin to make it case sensitive')
+;
+DROP INDEX idx1 ON t1;
+DROP TABLE t1;
+SET GLOBAL debug_dbug = @save_debug;
+CREATE TABLE t1 (
+value VARCHAR(1024)
+) ENGINE=InnoDB;
+CREATE FULLTEXT INDEX idx1 ON t1(value);
+Warnings:
+Warning	124	InnoDB rebuilding table to add column FTS_DOC_ID
+SET GLOBAL debug_dbug = '+d,fts_instrument_sync_request,fts_instrument_msg_sync_sleep';
+INSERT INTO t1 (value) VALUES
+('By default or with the IN NATURAL LANGUAGE MODE modifier'),
+('performs a natural language search for a string'),
+('collection is a set of one or more columns included'),
+('returns a relevance value; that is, a similarity measure'),
+('and the text in that row in the columns named in'),
+('By default, the search is performed in case-insensitive'),
+('sensitive full-text search, use a binary collation '),
+('example, a column that uses the latin1 character'),
+('collation of latin1_bin to make it case sensitive')
+;
+ALTER TABLE t1
+DROP INDEX idx1,
+ALGORITHM=INPLACE;
+DROP TABLE t1;
+SET GLOBAL debug_dbug = @save_debug;
+CREATE TABLE t1 (
+value VARCHAR(1024)
+) ENGINE=InnoDB;
+CREATE FULLTEXT INDEX idx1 ON t1(value);
+Warnings:
+Warning	124	InnoDB rebuilding table to add column FTS_DOC_ID
+SET GLOBAL debug_dbug = '+d,fts_instrument_sync_request,fts_instrument_msg_sync_sleep';
+INSERT INTO t1 (value) VALUES
+('example, a column that uses the latin1 character'),
+('collation of latin1_bin to make it case sensitive')
+;
+ALTER TABLE t1
+DROP INDEX idx1,
+ALGORITHM=COPY;
+DROP TABLE t1;
+SET GLOBAL debug_dbug = @save_debug;
+CREATE TABLE t1 (
+id1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+value VARCHAR(1024)
+) ENGINE=InnoDB;
+CREATE FULLTEXT INDEX idx1 ON t1(value);
+Warnings:
+Warning	124	InnoDB rebuilding table to add column FTS_DOC_ID
+SET GLOBAL debug_dbug = '+d,fts_instrument_sync_request,fts_instrument_msg_sync_sleep';
+INSERT INTO t1 (value) VALUES
+('example, a column that uses the latin1 character'),
+('collation of latin1_bin to make it case sensitive')
+;
+ALTER TABLE t1
+DROP COLUMN id1,
+ADD COLUMN id2 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+DROP INDEX idx1,
+ADD FULLTEXT INDEX idx2(value),
+ALGORITHM=INPLACE;
+DROP TABLE t1;
+SET GLOBAL debug_dbug = @save_debug;
diff --git a/mysql-test/suite/innodb_fts/t/sync_ddl.test b/mysql-test/suite/innodb_fts/t/sync_ddl.test
new file mode 100644
index 00000000000..2950297d5bb
--- /dev/null
+++ b/mysql-test/suite/innodb_fts/t/sync_ddl.test
@@ -0,0 +1,177 @@
+#
+# BUG#27082268 FTS synchronization issues
+#
+
+--source include/have_innodb.inc
+--source include/have_debug.inc
+
+#--------------------------------------
+# Check FTS_sync vs TRUNCATE (1)
+#--------------------------------------
+
+CREATE TABLE t1 (
+    id INT AUTO_INCREMENT NOT NULL PRIMARY KEY,
+    value VARCHAR(1024)
+    ) ENGINE=InnoDB;
+
+CREATE FULLTEXT INDEX idx1 ON t1(value);
+
+SET @save_debug = @@GLOBAL.debug_dbug;
+SET GLOBAL debug_dbug = '+d,fts_instrument_sync_request,fts_instrument_sync_before_syncing,ib_trunc_sleep_before_fts_cache_clear';
+
+INSERT INTO t1 (value) VALUES
+    ('By default or with the IN NATURAL LANGUAGE MODE modifier')
+    ;
+
+TRUNCATE TABLE t1;
+
+DROP TABLE t1;
+
+SET GLOBAL debug_dbug = @save_debug;
+
+#--------------------------------------
+# Check FTS sync vs DROP INDEX (2)
+#--------------------------------------
+
+CREATE TABLE t1 (
+    id INT AUTO_INCREMENT NOT NULL PRIMARY KEY,
+    value VARCHAR(1024)
+    ) ENGINE=InnoDB;
+
+CREATE FULLTEXT INDEX idx1 ON t1(value);
+
+SET GLOBAL debug_dbug = '+d,fts_instrument_sync_request,fts_instrument_write_words_before_select_index,ib_trunc_sleep_before_fts_cache_clear';
+
+INSERT INTO t1 (value) VALUES
+    ('By default or with the IN NATURAL LANGUAGE MODE modifier'),
+    ('performs a natural language search for a string'),
+    ('collection is a set of one or more columns included'),
+    ('returns a relevance value; that is, a similarity measure'),
+    ('and the text in that row in the columns named in'),
+    ('By default, the search is performed in case-insensitive'),
+    ('sensitive full-text search, use a binary collation '),
+    ('example, a column that uses the latin1 character'),
+    ('collation of latin1_bin to make it case sensitive')
+    ;
+
+TRUNCATE TABLE t1;
+
+DROP TABLE t1;
+
+SET GLOBAL debug_dbug = @save_debug;
+
+#--------------------------------------
+# Check FTS sync vs DROP INDEX
+#--------------------------------------
+
+CREATE TABLE t1 (
+    value VARCHAR(1024)
+    ) ENGINE=InnoDB;
+
+CREATE FULLTEXT INDEX idx1 ON t1(value);
+
+SET GLOBAL debug_dbug = '+d,fts_instrument_sync_request,fts_instrument_msg_sync_sleep';
+
+INSERT INTO t1 (value) VALUES
+    ('By default or with the IN NATURAL LANGUAGE MODE modifier'),
+    ('performs a natural language search for a string'),
+    ('collection is a set of one or more columns included'),
+    ('returns a relevance value; that is, a similarity measure'),
+    ('and the text in that row in the columns named in'),
+    ('By default, the search is performed in case-insensitive'),
+    ('sensitive full-text search, use a binary collation '),
+    ('example, a column that uses the latin1 character'),
+    ('collation of latin1_bin to make it case sensitive')
+    ;
+
+DROP INDEX idx1 ON t1;
+
+DROP TABLE t1;
+
+SET GLOBAL debug_dbug = @save_debug;
+
+#--------------------------------------
+# Check FTS sync vs ALTER TABLE DROP INDEX (INPLACE)
+#--------------------------------------
+
+CREATE TABLE t1 (
+    value VARCHAR(1024)
+    ) ENGINE=InnoDB;
+
+CREATE FULLTEXT INDEX idx1 ON t1(value);
+
+SET GLOBAL debug_dbug = '+d,fts_instrument_sync_request,fts_instrument_msg_sync_sleep';
+
+INSERT INTO t1 (value) VALUES
+    ('By default or with the IN NATURAL LANGUAGE MODE modifier'),
+    ('performs a natural language search for a string'),
+    ('collection is a set of one or more columns included'),
+    ('returns a relevance value; that is, a similarity measure'),
+    ('and the text in that row in the columns named in'),
+    ('By default, the search is performed in case-insensitive'),
+    ('sensitive full-text search, use a binary collation '),
+    ('example, a column that uses the latin1 character'),
+    ('collation of latin1_bin to make it case sensitive')
+    ;
+
+ALTER TABLE t1
+    DROP INDEX idx1,
+    ALGORITHM=INPLACE;
+
+DROP TABLE t1;
+
+SET GLOBAL debug_dbug = @save_debug;
+
+#--------------------------------------
+# Check FTS sync vs ALTER TABLE DROP INDEX (COPY)
+#--------------------------------------
+
+CREATE TABLE t1 (
+    value VARCHAR(1024)
+    ) ENGINE=InnoDB;
+
+CREATE FULLTEXT INDEX idx1 ON t1(value);
+
+SET GLOBAL debug_dbug = '+d,fts_instrument_sync_request,fts_instrument_msg_sync_sleep';
+
+INSERT INTO t1 (value) VALUES
+    ('example, a column that uses the latin1 character'),
+    ('collation of latin1_bin to make it case sensitive')
+    ;
+
+ALTER TABLE t1
+    DROP INDEX idx1,
+    ALGORITHM=COPY;
+
+DROP TABLE t1;
+
+SET GLOBAL debug_dbug = @save_debug;
+
+#--------------------------------------
+# Check FTS sync vs ALTER TABLE (INPLACE, new cluster)
+#--------------------------------------
+
+CREATE TABLE t1 (
+    id1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+    value VARCHAR(1024)
+    ) ENGINE=InnoDB;
+
+CREATE FULLTEXT INDEX idx1 ON t1(value);
+
+SET GLOBAL debug_dbug = '+d,fts_instrument_sync_request,fts_instrument_msg_sync_sleep';
+
+INSERT INTO t1 (value) VALUES
+    ('example, a column that uses the latin1 character'),
+    ('collation of latin1_bin to make it case sensitive')
+    ;
+
+ALTER TABLE t1
+    DROP COLUMN id1,
+    ADD COLUMN id2 INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+    DROP INDEX idx1,
+    ADD FULLTEXT INDEX idx2(value),
+    ALGORITHM=INPLACE;
+
+DROP TABLE t1;
+
+SET GLOBAL debug_dbug = @save_debug;
diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc
index 86dd76bbf49..6044abd8e79 100644
--- a/storage/innobase/fts/fts0fts.cc
+++ b/storage/innobase/fts/fts0fts.cc
@@ -268,7 +268,7 @@ FTS auxiliary INDEX table and clear the cache at the end.
 @param[in,out]	sync		sync state
 @param[in]	unlock_cache	whether unlock cache lock when write node
 @param[in]	wait		whether wait when a sync is in progress
-@param[in]      has_dict        whether has dict operation lock
+@param[in]	has_dict	whether has dict operation lock
 @return DB_SUCCESS if all OK */
 static
 dberr_t
@@ -3961,6 +3961,9 @@ fts_sync_write_words(
 
 		word = rbt_value(fts_tokenizer_word_t, rbt_node);
 
+		DBUG_EXECUTE_IF("fts_instrument_write_words_before_select_index",
+				os_thread_sleep(300000););
+
 		selected = fts_select_index(
 			index_cache->charset, word->text.f_str,
 			word->text.f_len);
@@ -4525,7 +4528,7 @@ FTS auxiliary INDEX table and clear the cache at the end.
 @param[in,out]	sync		sync state
 @param[in]	unlock_cache	whether unlock cache lock when write node
 @param[in]	wait		whether wait when a sync is in progress
-@param[in]      has_dict        whether has dict operation lock
+@param[in]	has_dict	whether has dict operation lock
 @return DB_SUCCESS if all OK */
 static
 dberr_t
@@ -4587,15 +4590,13 @@ begin_sync:
 			continue;
 		}
 
+		DBUG_EXECUTE_IF("fts_instrument_sync_before_syncing",
+				os_thread_sleep(300000););
 		index_cache->index->index_fts_syncing = true;
-		DBUG_EXECUTE_IF("fts_instrument_sync_sleep_drop_waits",
-				os_thread_sleep(10000000);
-				);
 
 		error = fts_sync_index(sync, index_cache);
 
-		if (error != DB_SUCCESS && !sync->interrupted) {
-
+		if (error != DB_SUCCESS) {
 			goto end_sync;
 		}
 	}
@@ -4630,8 +4631,8 @@ end_sync:
 	}
 
 	rw_lock_x_lock(&cache->lock);
-	/* Clear fts syncing flags of any indexes incase sync is
-	interrupeted */
+	/* Clear fts syncing flags of any indexes in case sync is
+	interrupted */
 	for (i = 0; i < ib_vector_size(cache->indexes); ++i) {
 		static_cast<fts_index_cache_t*>(
 			ib_vector_get(cache->indexes, i))
diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc
index d9f96948000..77293bc867a 100644
--- a/storage/innobase/fts/fts0opt.cc
+++ b/storage/innobase/fts/fts0opt.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2017, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2007, 2018, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2016, MariaDB Corporation. All Rights reserved.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -58,12 +58,6 @@ static const ulint FTS_WORD_NODES_INIT_SIZE = 64;
 /** Last time we did check whether system need a sync */
 static ib_time_t	last_check_sync_time;
 
-#if 0
-/** Check each table in round robin to see whether they'd
-need to be "optimized" */
-static	ulint	fts_optimize_sync_iterator = 0;
-#endif
-
 /** State of a table within the optimization sub system. */
 enum fts_state_t {
 	FTS_STATE_LOADED,
@@ -75,17 +69,11 @@ enum fts_state_t {
 
 /** FTS optimize thread message types. */
 enum fts_msg_type_t {
-	FTS_MSG_START,			/*!< Start optimizing thread */
-
-	FTS_MSG_PAUSE,			/*!< Pause optimizing thread */
-
 	FTS_MSG_STOP,			/*!< Stop optimizing and exit thread */
 
 	FTS_MSG_ADD_TABLE,		/*!< Add table to the optimize thread's
 					work queue */
 
-	FTS_MSG_OPTIMIZE_TABLE,		/*!< Optimize a table */
-
 	FTS_MSG_DEL_TABLE,		/*!< Remove a table from the optimize
 					threads work queue */
 	FTS_MSG_SYNC_TABLE		/*!< Sync fts cache of a table */
@@ -235,7 +223,7 @@ struct fts_msg_t {
 /** The number of words to read and optimize in a single pass. */
 UNIV_INTERN ulong	fts_num_word_optimize;
 
-// FIXME
+/** Whether to enable additional FTS diagnostic printout. */
 UNIV_INTERN char	fts_enable_diag_print;
 
 /** ZLib compressed block size.*/
@@ -2560,13 +2548,9 @@ fts_optimize_create_msg(
 	return(msg);
 }
 
-/**********************************************************************//**
-Add the table to add to the OPTIMIZER's list. */
-UNIV_INTERN
-void
-fts_optimize_add_table(
-/*===================*/
-	dict_table_t*	table)			/*!< in: table to add */
+/** Add the table to add to the OPTIMIZER's list.
+@param[in]	table	table to add */
+UNIV_INTERN void fts_optimize_add_table(dict_table_t* table)
 {
 	fts_msg_t*	msg;
 
@@ -2584,26 +2568,6 @@ fts_optimize_add_table(
 	ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
 }
 
-/**********************************************************************//**
-Optimize a table. */
-UNIV_INTERN
-void
-fts_optimize_do_table(
-/*==================*/
-	dict_table_t*	table)			/*!< in: table to optimize */
-{
-	fts_msg_t*	msg;
-
-	/* Optimizer thread could be shutdown */
-	if (!fts_optimize_wq) {
-		return;
-	}
-
-	msg = fts_optimize_create_msg(FTS_MSG_OPTIMIZE_TABLE, table);
-
-	ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
-}
-
 /**********************************************************************//**
 Remove the table from the OPTIMIZER's list. We do wait for
 acknowledgement from the consumer of the message. */
@@ -2618,7 +2582,7 @@ fts_optimize_remove_table(
 	fts_msg_del_t*	remove;
 
 	/* if the optimize system not yet initialized, return */
-	if (!fts_optimize_wq) {
+	if (!fts_optimize_is_init()) {
 		return;
 	}
 
@@ -2660,7 +2624,7 @@ fts_optimize_request_sync_table(
 	table_id_t*	table_id;
 
 	/* if the optimize system not yet initialized, return */
-	if (!fts_optimize_wq) {
+	if (!fts_optimize_is_init()) {
 		return;
 	}
 
@@ -2682,54 +2646,6 @@ fts_optimize_request_sync_table(
 	ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
 }
 
-/**********************************************************************//**
-Find the slot for a particular table.
-@return slot if found else NULL. */
-static
-fts_slot_t*
-fts_optimize_find_slot(
-/*===================*/
-	ib_vector_t*		tables,		/*!< in: vector of tables */
-	const dict_table_t*	table)		/*!< in: table to add */
-{
-	ulint		i;
-
-	for (i = 0; i < ib_vector_size(tables); ++i) {
-		fts_slot_t*	slot;
-
-		slot = static_cast<fts_slot_t*>(ib_vector_get(tables, i));
-
-		if (slot->table->id == table->id) {
-			return(slot);
-		}
-	}
-
-	return(NULL);
-}
-
-/**********************************************************************//**
-Start optimizing table. */
-static
-void
-fts_optimize_start_table(
-/*=====================*/
-	ib_vector_t*		tables,		/*!< in/out: vector of tables */
-	dict_table_t*		table)		/*!< in: table to optimize */
-{
-	fts_slot_t*	slot;
-
-	slot = fts_optimize_find_slot(tables, table);
-
-	if (slot == NULL) {
-		ut_print_timestamp(stderr);
-		fprintf(stderr, " InnoDB: Error: table %s not registered "
-			"with the optimize thread.\n", table->name);
-	} else {
-		slot->last_run = 0;
-		slot->completed = 0;
-	}
-}
-
 /**********************************************************************//**
 Add the table to the vector if it doesn't already exist. */
 static
@@ -2912,57 +2828,6 @@ fts_is_sync_needed(
 	return(false);
 }
 
-#if 0
-/*********************************************************************//**
-Check whether a table needs to be optimized. */
-static
-void
-fts_optimize_need_sync(
-/*===================*/
-	ib_vector_t*	tables)	/*!< in: list of tables */
-{
-	dict_table_t*	table = NULL;
-	fts_slot_t*	slot;
-	ulint		num_table = ib_vector_size(tables);
-
-	if (!num_table) {
-		return;
-	}
-
-	if (fts_optimize_sync_iterator >= num_table) {
-		fts_optimize_sync_iterator = 0;
-	}
-
-	slot = ib_vector_get(tables, fts_optimize_sync_iterator);
-	table = slot->table;
-
-	if (!table) {
-		return;
-	}
-
-	ut_ad(table->fts);
-
-	if (table->fts->cache) {
-		ulint	deleted = table->fts->cache->deleted;
-
-		if (table->fts->cache->added
-		    >= fts_optimize_add_threshold) {
-			fts_sync_table(table);
-		} else if (deleted >= fts_optimize_delete_threshold) {
-			fts_optimize_do_table(table);
-
-			mutex_enter(&table->fts->cache->deleted_lock);
-			table->fts->cache->deleted -= deleted;
-			mutex_exit(&table->fts->cache->deleted_lock);
-		}
-	}
-
-	fts_optimize_sync_iterator++;
-
-	return;
-}
-#endif
-
 /** Sync fts cache of a table
 @param[in]	table_id	table id */
 void
@@ -2975,7 +2840,7 @@ fts_optimize_sync_table(
 
 	if (table) {
 		if (dict_table_has_fts_index(table) && table->fts->cache) {
-			fts_sync_table(table, true, false, true);
+			fts_sync_table(table, true, false, false);
 		}
 
 		dict_table_close(table, FALSE, FALSE);
@@ -3047,8 +2912,7 @@ fts_optimize_thread(
 			fts_msg_t*	msg;
 
 			msg = static_cast<fts_msg_t*>(
-				ib_wqueue_timedwait(wq,
-						    FTS_QUEUE_WAIT_IN_USECS));
+				ib_wqueue_timedwait(wq, FTS_QUEUE_WAIT_IN_USECS));
 
 			/* Timeout ? */
 			if (msg == NULL) {
@@ -3060,12 +2924,6 @@ fts_optimize_thread(
 			}
 
 			switch (msg->type) {
-			case FTS_MSG_START:
-				break;
-
-			case FTS_MSG_PAUSE:
-				break;
-
 			case FTS_MSG_STOP:
 				done = TRUE;
 				exit_event = (os_event_t) msg->ptr;
@@ -3081,15 +2939,6 @@ fts_optimize_thread(
 				}
 				break;
 
-			case FTS_MSG_OPTIMIZE_TABLE:
-				if (!done) {
-					fts_optimize_start_table(
-						tables,
-						static_cast<dict_table_t*>(
-						msg->ptr));
-				}
-				break;
-
 			case FTS_MSG_DEL_TABLE:
 				if (fts_optimize_del_table(
 					tables, static_cast<fts_msg_del_t*>(
@@ -3104,6 +2953,10 @@ fts_optimize_thread(
 				break;
 
 			case FTS_MSG_SYNC_TABLE:
+				DBUG_EXECUTE_IF(
+					"fts_instrument_msg_sync_sleep",
+					os_thread_sleep(300000););
+
 				fts_optimize_sync_table(
 					*static_cast<table_id_t*>(msg->ptr));
 				break;
@@ -3163,7 +3016,7 @@ fts_optimize_init(void)
 	ut_ad(!srv_read_only_mode);
 
 	/* For now we only support one optimize thread. */
-	ut_a(fts_optimize_wq == NULL);
+	ut_a(!fts_optimize_is_init());
 
 	fts_optimize_wq = ib_wqueue_create();
 	ut_a(fts_optimize_wq != NULL);
diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc
index 3de46ed4f80..6a3e9575eb1 100644
--- a/storage/innobase/handler/handler0alter.cc
+++ b/storage/innobase/handler/handler0alter.cc
@@ -5710,24 +5710,24 @@ ha_innobase::commit_inplace_alter_table(
 	trx_t*		trx		= ctx0->trx;
 	bool		fail		= false;
 
-	if (new_clustered) {
-		for (inplace_alter_handler_ctx** pctx = ctx_array;
-		     *pctx; pctx++) {
-			ha_innobase_inplace_ctx*	ctx
-				= static_cast<ha_innobase_inplace_ctx*>(*pctx);
-			DBUG_ASSERT(ctx->need_rebuild());
+	/* Stop background FTS operations. */
+	for (inplace_alter_handler_ctx** pctx = ctx_array;
+			 *pctx; pctx++) {
+		ha_innobase_inplace_ctx*	ctx
+			= static_cast<ha_innobase_inplace_ctx*>(*pctx);
 
+		DBUG_ASSERT(new_clustered == ctx->need_rebuild());
+
+		if (new_clustered) {
 			if (ctx->old_table->fts) {
 				ut_ad(!ctx->old_table->fts->add_wq);
-				fts_optimize_remove_table(
-					ctx->old_table);
+				fts_optimize_remove_table(ctx->old_table);
 			}
+		}
 
-			if (ctx->new_table->fts) {
-				ut_ad(!ctx->new_table->fts->add_wq);
-				fts_optimize_remove_table(
-					ctx->new_table);
-			}
+		if (ctx->new_table->fts) {
+			ut_ad(!ctx->new_table->fts->add_wq);
+			fts_optimize_remove_table(ctx->new_table);
 		}
 	}
 
@@ -5772,41 +5772,40 @@ ha_innobase::commit_inplace_alter_table(
 
 	/* Make a concurrent Drop fts Index to wait until sync of that
 	fts index is happening in the background */
-	for (;;) {
+	for (int retry_count = 0;;) {
 		bool    retry = false;
 
 		for (inplace_alter_handler_ctx** pctx = ctx_array;
 		    *pctx; pctx++) {
-			int count =0;
 			ha_innobase_inplace_ctx*        ctx
 				= static_cast<ha_innobase_inplace_ctx*>(*pctx);
 			DBUG_ASSERT(new_clustered == ctx->need_rebuild());
 
 			if (dict_fts_index_syncing(ctx->old_table)) {
-				count++;
-				if (count == 100) {
-					fprintf(stderr,
-					"Drop index waiting for background sync"
-					"to finish\n");
-				}
 				retry = true;
+				break;
 			}
 
 			if (new_clustered && dict_fts_index_syncing(ctx->new_table)) {
-				count++;
-				if (count == 100) {
-					fprintf(stderr,
-                                        "Drop index waiting for background sync"
-                                        "to finish\n");
-				}
 				retry = true;
+				break;
 			}
 		}
 
-		 if (!retry) {
+		if (!retry) {
 			 break;
 		}
 
+		/* Print a message if waiting for a long time. */
+		if (retry_count < 100) {
+			retry_count++;
+		} else {
+			ib_logf(IB_LOG_LEVEL_INFO,
+				"Drop index waiting for background sync"
+				" to finish");
+			retry_count = 0;
+		}
+
 		DICT_BG_YIELD(trx);
 	}
 
@@ -6070,6 +6069,11 @@ foreign_fail:
 			ut_a(fts_check_cached_index(ctx->old_table));
 			DBUG_INJECT_CRASH("ib_commit_inplace_crash_fail",
 					  crash_fail_inject_count++);
+
+			/* Restart the FTS background operations. */
+			if (ctx->old_table->fts) {
+				fts_optimize_add_table(ctx->old_table);
+			}
 		}
 
 		row_mysql_unlock_data_dictionary(trx);
@@ -6118,8 +6122,6 @@ foreign_fail:
 			dict_table_autoinc_unlock(t);
 		}
 
-		bool	add_fts	= false;
-
 		/* Publish the created fulltext index, if any.
 		Note that a fulltext index can be created without
 		creating the clustered index, if there already exists
@@ -6134,14 +6136,14 @@ foreign_fail:
 				is left unset when a drop proceeds the add. */
 				DICT_TF2_FLAG_SET(ctx->new_table, DICT_TF2_FTS);
 				fts_add_index(index, ctx->new_table);
-				add_fts = true;
 			}
 		}
 
 		ut_d(dict_table_check_for_dup_indexes(
 			     ctx->new_table, CHECK_ALL_COMPLETE));
 
-		if (add_fts) {
+		/* Start/Restart the FTS background operations. */
+		if (ctx->new_table->fts) {
 			fts_optimize_add_table(ctx->new_table);
 		}
 
diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h
index cd94956dc55..4c2d247e0a6 100644
--- a/storage/innobase/include/fts0fts.h
+++ b/storage/innobase/include/fts0fts.h
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2011, 2018, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2016, 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -705,6 +705,12 @@ fts_drop_index_tables(
 	dict_index_t*	index)			/*!< in: Index to drop */
 	MY_ATTRIBUTE((nonnull, warn_unused_result));
 
+/** Add the table to add to the OPTIMIZER's list.
+@param[in]	table	table to add */
+void
+fts_optimize_add_table(
+	dict_table_t*	table);
+
 /******************************************************************//**
 Remove the table from the OPTIMIZER's list. We do wait for
 acknowledgement from the consumer of the message. */
diff --git a/storage/innobase/include/fts0priv.h b/storage/innobase/include/fts0priv.h
index 2d4e9d88fd1..a3936f54a48 100644
--- a/storage/innobase/include/fts0priv.h
+++ b/storage/innobase/include/fts0priv.h
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2011, 2018, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -598,22 +598,6 @@ fts_get_table_id(
 					long */
 	MY_ATTRIBUTE((nonnull, warn_unused_result));
 /******************************************************************//**
-Add the table to add to the OPTIMIZER's list. */
-UNIV_INTERN
-void
-fts_optimize_add_table(
-/*===================*/
-	dict_table_t*	table)		/*!< in: table to add */
-	MY_ATTRIBUTE((nonnull));
-/******************************************************************//**
-Optimize a table. */
-UNIV_INTERN
-void
-fts_optimize_do_table(
-/*==================*/
-	dict_table_t*	table)		/*!< in: table to optimize */
-	MY_ATTRIBUTE((nonnull));
-/******************************************************************//**
 Construct the prefix name of an FTS table.
 @return own: table name, must be freed with mem_free() */
 UNIV_INTERN
diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc
index bb1a27be4ba..87d40ee2ba4 100644
--- a/storage/innobase/row/row0mysql.cc
+++ b/storage/innobase/row/row0mysql.cc
@@ -3309,6 +3309,10 @@ row_truncate_table_for_mysql(
 		return(DB_TABLESPACE_NOT_FOUND);
 	}
 
+	if (table->fts) {
+		fts_optimize_remove_table(table);
+	}
+
 	trx_start_for_ddl(trx, TRX_DICT_OP_TABLE);
 
 	trx->op_info = "truncating table";
@@ -3709,6 +3713,9 @@ next_rec:
 
 		/* Reset the Doc ID in cache to 0 */
 		if (has_internal_doc_id && table->fts->cache) {
+			DBUG_EXECUTE_IF("ib_trunc_sleep_before_fts_cache_clear",
+					os_thread_sleep(10000000););
+
 			table->fts->fts_status |= TABLE_DICT_LOCKED;
 			fts_update_next_doc_id(trx, table, NULL, 0);
 			fts_cache_clear(table->fts->cache);
@@ -3732,6 +3739,11 @@ funct_exit:
                 table->memcached_sync_count = 0;
         }
 
+	/* Add the table back to FTS optimize background thread. */
+	if (table->fts) {
+		fts_optimize_add_table(table);
+	}
+
 	row_mysql_unlock_data_dictionary(trx);
 
 	dict_stats_update(table, DICT_STATS_EMPTY_TABLE);
diff --git a/storage/xtradb/fts/fts0fts.cc b/storage/xtradb/fts/fts0fts.cc
index 36a1d234bf4..2f9e331219b 100644
--- a/storage/xtradb/fts/fts0fts.cc
+++ b/storage/xtradb/fts/fts0fts.cc
@@ -268,7 +268,7 @@ FTS auxiliary INDEX table and clear the cache at the end.
 @param[in,out]	sync		sync state
 @param[in]	unlock_cache	whether unlock cache lock when write node
 @param[in]	wait		whether wait when a sync is in progress
-@param[in]      has_dict        whether has dict operation lock
+@param[in]	has_dict	whether has dict operation lock
 @return DB_SUCCESS if all OK */
 static
 dberr_t
@@ -3961,6 +3961,9 @@ fts_sync_write_words(
 
 		word = rbt_value(fts_tokenizer_word_t, rbt_node);
 
+		DBUG_EXECUTE_IF("fts_instrument_write_words_before_select_index",
+				os_thread_sleep(300000););
+
 		selected = fts_select_index(
 			index_cache->charset, word->text.f_str,
 			word->text.f_len);
@@ -4525,7 +4528,7 @@ FTS auxiliary INDEX table and clear the cache at the end.
 @param[in,out]	sync		sync state
 @param[in]	unlock_cache	whether unlock cache lock when write node
 @param[in]	wait		whether wait when a sync is in progress
-@param[in]      has_dict        whether has dict operation lock
+@param[in]	has_dict	whether has dict operation lock
 @return DB_SUCCESS if all OK */
 static
 dberr_t
@@ -4587,15 +4590,13 @@ begin_sync:
 			continue;
 		}
 
+		DBUG_EXECUTE_IF("fts_instrument_sync_before_syncing",
+				os_thread_sleep(300000););
 		index_cache->index->index_fts_syncing = true;
-		DBUG_EXECUTE_IF("fts_instrument_sync_sleep_drop_waits",
-				os_thread_sleep(10000000);
-				);
 
 		error = fts_sync_index(sync, index_cache);
 
-		if (error != DB_SUCCESS && !sync->interrupted) {
-
+		if (error != DB_SUCCESS) {
 			goto end_sync;
 		}
 	}
@@ -4630,8 +4631,8 @@ end_sync:
 	}
 
 	rw_lock_x_lock(&cache->lock);
-	/* Clear fts syncing flags of any indexes incase sync is
-	interrupeted */
+	/* Clear fts syncing flags of any indexes in case sync is
+	interrupted */
 	for (i = 0; i < ib_vector_size(cache->indexes); ++i) {
 		static_cast<fts_index_cache_t*>(
 			ib_vector_get(cache->indexes, i))
diff --git a/storage/xtradb/fts/fts0opt.cc b/storage/xtradb/fts/fts0opt.cc
index d9f96948000..77293bc867a 100644
--- a/storage/xtradb/fts/fts0opt.cc
+++ b/storage/xtradb/fts/fts0opt.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2017, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2007, 2018, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2016, MariaDB Corporation. All Rights reserved.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -58,12 +58,6 @@ static const ulint FTS_WORD_NODES_INIT_SIZE = 64;
 /** Last time we did check whether system need a sync */
 static ib_time_t	last_check_sync_time;
 
-#if 0
-/** Check each table in round robin to see whether they'd
-need to be "optimized" */
-static	ulint	fts_optimize_sync_iterator = 0;
-#endif
-
 /** State of a table within the optimization sub system. */
 enum fts_state_t {
 	FTS_STATE_LOADED,
@@ -75,17 +69,11 @@ enum fts_state_t {
 
 /** FTS optimize thread message types. */
 enum fts_msg_type_t {
-	FTS_MSG_START,			/*!< Start optimizing thread */
-
-	FTS_MSG_PAUSE,			/*!< Pause optimizing thread */
-
 	FTS_MSG_STOP,			/*!< Stop optimizing and exit thread */
 
 	FTS_MSG_ADD_TABLE,		/*!< Add table to the optimize thread's
 					work queue */
 
-	FTS_MSG_OPTIMIZE_TABLE,		/*!< Optimize a table */
-
 	FTS_MSG_DEL_TABLE,		/*!< Remove a table from the optimize
 					threads work queue */
 	FTS_MSG_SYNC_TABLE		/*!< Sync fts cache of a table */
@@ -235,7 +223,7 @@ struct fts_msg_t {
 /** The number of words to read and optimize in a single pass. */
 UNIV_INTERN ulong	fts_num_word_optimize;
 
-// FIXME
+/** Whether to enable additional FTS diagnostic printout. */
 UNIV_INTERN char	fts_enable_diag_print;
 
 /** ZLib compressed block size.*/
@@ -2560,13 +2548,9 @@ fts_optimize_create_msg(
 	return(msg);
 }
 
-/**********************************************************************//**
-Add the table to add to the OPTIMIZER's list. */
-UNIV_INTERN
-void
-fts_optimize_add_table(
-/*===================*/
-	dict_table_t*	table)			/*!< in: table to add */
+/** Add the table to add to the OPTIMIZER's list.
+@param[in]	table	table to add */
+UNIV_INTERN void fts_optimize_add_table(dict_table_t* table)
 {
 	fts_msg_t*	msg;
 
@@ -2584,26 +2568,6 @@ fts_optimize_add_table(
 	ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
 }
 
-/**********************************************************************//**
-Optimize a table. */
-UNIV_INTERN
-void
-fts_optimize_do_table(
-/*==================*/
-	dict_table_t*	table)			/*!< in: table to optimize */
-{
-	fts_msg_t*	msg;
-
-	/* Optimizer thread could be shutdown */
-	if (!fts_optimize_wq) {
-		return;
-	}
-
-	msg = fts_optimize_create_msg(FTS_MSG_OPTIMIZE_TABLE, table);
-
-	ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
-}
-
 /**********************************************************************//**
 Remove the table from the OPTIMIZER's list. We do wait for
 acknowledgement from the consumer of the message. */
@@ -2618,7 +2582,7 @@ fts_optimize_remove_table(
 	fts_msg_del_t*	remove;
 
 	/* if the optimize system not yet initialized, return */
-	if (!fts_optimize_wq) {
+	if (!fts_optimize_is_init()) {
 		return;
 	}
 
@@ -2660,7 +2624,7 @@ fts_optimize_request_sync_table(
 	table_id_t*	table_id;
 
 	/* if the optimize system not yet initialized, return */
-	if (!fts_optimize_wq) {
+	if (!fts_optimize_is_init()) {
 		return;
 	}
 
@@ -2682,54 +2646,6 @@ fts_optimize_request_sync_table(
 	ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
 }
 
-/**********************************************************************//**
-Find the slot for a particular table.
-@return slot if found else NULL. */
-static
-fts_slot_t*
-fts_optimize_find_slot(
-/*===================*/
-	ib_vector_t*		tables,		/*!< in: vector of tables */
-	const dict_table_t*	table)		/*!< in: table to add */
-{
-	ulint		i;
-
-	for (i = 0; i < ib_vector_size(tables); ++i) {
-		fts_slot_t*	slot;
-
-		slot = static_cast<fts_slot_t*>(ib_vector_get(tables, i));
-
-		if (slot->table->id == table->id) {
-			return(slot);
-		}
-	}
-
-	return(NULL);
-}
-
-/**********************************************************************//**
-Start optimizing table. */
-static
-void
-fts_optimize_start_table(
-/*=====================*/
-	ib_vector_t*		tables,		/*!< in/out: vector of tables */
-	dict_table_t*		table)		/*!< in: table to optimize */
-{
-	fts_slot_t*	slot;
-
-	slot = fts_optimize_find_slot(tables, table);
-
-	if (slot == NULL) {
-		ut_print_timestamp(stderr);
-		fprintf(stderr, " InnoDB: Error: table %s not registered "
-			"with the optimize thread.\n", table->name);
-	} else {
-		slot->last_run = 0;
-		slot->completed = 0;
-	}
-}
-
 /**********************************************************************//**
 Add the table to the vector if it doesn't already exist. */
 static
@@ -2912,57 +2828,6 @@ fts_is_sync_needed(
 	return(false);
 }
 
-#if 0
-/*********************************************************************//**
-Check whether a table needs to be optimized. */
-static
-void
-fts_optimize_need_sync(
-/*===================*/
-	ib_vector_t*	tables)	/*!< in: list of tables */
-{
-	dict_table_t*	table = NULL;
-	fts_slot_t*	slot;
-	ulint		num_table = ib_vector_size(tables);
-
-	if (!num_table) {
-		return;
-	}
-
-	if (fts_optimize_sync_iterator >= num_table) {
-		fts_optimize_sync_iterator = 0;
-	}
-
-	slot = ib_vector_get(tables, fts_optimize_sync_iterator);
-	table = slot->table;
-
-	if (!table) {
-		return;
-	}
-
-	ut_ad(table->fts);
-
-	if (table->fts->cache) {
-		ulint	deleted = table->fts->cache->deleted;
-
-		if (table->fts->cache->added
-		    >= fts_optimize_add_threshold) {
-			fts_sync_table(table);
-		} else if (deleted >= fts_optimize_delete_threshold) {
-			fts_optimize_do_table(table);
-
-			mutex_enter(&table->fts->cache->deleted_lock);
-			table->fts->cache->deleted -= deleted;
-			mutex_exit(&table->fts->cache->deleted_lock);
-		}
-	}
-
-	fts_optimize_sync_iterator++;
-
-	return;
-}
-#endif
-
 /** Sync fts cache of a table
 @param[in]	table_id	table id */
 void
@@ -2975,7 +2840,7 @@ fts_optimize_sync_table(
 
 	if (table) {
 		if (dict_table_has_fts_index(table) && table->fts->cache) {
-			fts_sync_table(table, true, false, true);
+			fts_sync_table(table, true, false, false);
 		}
 
 		dict_table_close(table, FALSE, FALSE);
@@ -3047,8 +2912,7 @@ fts_optimize_thread(
 			fts_msg_t*	msg;
 
 			msg = static_cast<fts_msg_t*>(
-				ib_wqueue_timedwait(wq,
-						    FTS_QUEUE_WAIT_IN_USECS));
+				ib_wqueue_timedwait(wq, FTS_QUEUE_WAIT_IN_USECS));
 
 			/* Timeout ? */
 			if (msg == NULL) {
@@ -3060,12 +2924,6 @@ fts_optimize_thread(
 			}
 
 			switch (msg->type) {
-			case FTS_MSG_START:
-				break;
-
-			case FTS_MSG_PAUSE:
-				break;
-
 			case FTS_MSG_STOP:
 				done = TRUE;
 				exit_event = (os_event_t) msg->ptr;
@@ -3081,15 +2939,6 @@ fts_optimize_thread(
 				}
 				break;
 
-			case FTS_MSG_OPTIMIZE_TABLE:
-				if (!done) {
-					fts_optimize_start_table(
-						tables,
-						static_cast<dict_table_t*>(
-						msg->ptr));
-				}
-				break;
-
 			case FTS_MSG_DEL_TABLE:
 				if (fts_optimize_del_table(
 					tables, static_cast<fts_msg_del_t*>(
@@ -3104,6 +2953,10 @@ fts_optimize_thread(
 				break;
 
 			case FTS_MSG_SYNC_TABLE:
+				DBUG_EXECUTE_IF(
+					"fts_instrument_msg_sync_sleep",
+					os_thread_sleep(300000););
+
 				fts_optimize_sync_table(
 					*static_cast<table_id_t*>(msg->ptr));
 				break;
@@ -3163,7 +3016,7 @@ fts_optimize_init(void)
 	ut_ad(!srv_read_only_mode);
 
 	/* For now we only support one optimize thread. */
-	ut_a(fts_optimize_wq == NULL);
+	ut_a(!fts_optimize_is_init());
 
 	fts_optimize_wq = ib_wqueue_create();
 	ut_a(fts_optimize_wq != NULL);
diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc
index 151c3237b2b..bc6a6dd68d4 100644
--- a/storage/xtradb/handler/handler0alter.cc
+++ b/storage/xtradb/handler/handler0alter.cc
@@ -5726,24 +5726,24 @@ ha_innobase::commit_inplace_alter_table(
 	trx_t*		trx		= ctx0->trx;
 	bool		fail		= false;
 
-	if (new_clustered) {
-		for (inplace_alter_handler_ctx** pctx = ctx_array;
-		     *pctx; pctx++) {
-			ha_innobase_inplace_ctx*	ctx
-				= static_cast<ha_innobase_inplace_ctx*>(*pctx);
-			DBUG_ASSERT(ctx->need_rebuild());
+	/* Stop background FTS operations. */
+	for (inplace_alter_handler_ctx** pctx = ctx_array;
+			 *pctx; pctx++) {
+		ha_innobase_inplace_ctx*	ctx
+			= static_cast<ha_innobase_inplace_ctx*>(*pctx);
 
+		DBUG_ASSERT(new_clustered == ctx->need_rebuild());
+
+		if (new_clustered) {
 			if (ctx->old_table->fts) {
 				ut_ad(!ctx->old_table->fts->add_wq);
-				fts_optimize_remove_table(
-					ctx->old_table);
+				fts_optimize_remove_table(ctx->old_table);
 			}
+		}
 
-			if (ctx->new_table->fts) {
-				ut_ad(!ctx->new_table->fts->add_wq);
-				fts_optimize_remove_table(
-					ctx->new_table);
-			}
+		if (ctx->new_table->fts) {
+			ut_ad(!ctx->new_table->fts->add_wq);
+			fts_optimize_remove_table(ctx->new_table);
 		}
 	}
 
@@ -5788,41 +5788,40 @@ ha_innobase::commit_inplace_alter_table(
 
 	/* Make a concurrent Drop fts Index to wait until sync of that
 	fts index is happening in the background */
-	for (;;) {
+	for (int retry_count = 0;;) {
 		bool    retry = false;
 
 		for (inplace_alter_handler_ctx** pctx = ctx_array;
 		    *pctx; pctx++) {
-			int count =0;
 			ha_innobase_inplace_ctx*        ctx
 				= static_cast<ha_innobase_inplace_ctx*>(*pctx);
 			DBUG_ASSERT(new_clustered == ctx->need_rebuild());
 
 			if (dict_fts_index_syncing(ctx->old_table)) {
-				count++;
-				if (count == 100) {
-					fprintf(stderr,
-					"Drop index waiting for background sync"
-					"to finish\n");
-				}
 				retry = true;
+				break;
 			}
 
 			if (new_clustered && dict_fts_index_syncing(ctx->new_table)) {
-				count++;
-				if (count == 100) {
-					fprintf(stderr,
-                                        "Drop index waiting for background sync"
-                                        "to finish\n");
-				}
 				retry = true;
+				break;
 			}
 		}
 
-		 if (!retry) {
+		if (!retry) {
 			 break;
 		}
 
+		/* Print a message if waiting for a long time. */
+		if (retry_count < 100) {
+			retry_count++;
+		} else {
+			ib_logf(IB_LOG_LEVEL_INFO,
+				"Drop index waiting for background sync"
+				" to finish");
+			retry_count = 0;
+		}
+
 		DICT_BG_YIELD(trx);
 	}
 
@@ -6086,6 +6085,11 @@ foreign_fail:
 			ut_a(fts_check_cached_index(ctx->old_table));
 			DBUG_INJECT_CRASH("ib_commit_inplace_crash_fail",
 					  crash_fail_inject_count++);
+
+			/* Restart the FTS background operations. */
+			if (ctx->old_table->fts) {
+				fts_optimize_add_table(ctx->old_table);
+			}
 		}
 
 		row_mysql_unlock_data_dictionary(trx);
@@ -6134,8 +6138,6 @@ foreign_fail:
 			dict_table_autoinc_unlock(t);
 		}
 
-		bool	add_fts	= false;
-
 		/* Publish the created fulltext index, if any.
 		Note that a fulltext index can be created without
 		creating the clustered index, if there already exists
@@ -6150,14 +6152,14 @@ foreign_fail:
 				is left unset when a drop proceeds the add. */
 				DICT_TF2_FLAG_SET(ctx->new_table, DICT_TF2_FTS);
 				fts_add_index(index, ctx->new_table);
-				add_fts = true;
 			}
 		}
 
 		ut_d(dict_table_check_for_dup_indexes(
 			     ctx->new_table, CHECK_ALL_COMPLETE));
 
-		if (add_fts) {
+		/* Start/Restart the FTS background operations. */
+		if (ctx->new_table->fts) {
 			fts_optimize_add_table(ctx->new_table);
 		}
 
diff --git a/storage/xtradb/include/fts0fts.h b/storage/xtradb/include/fts0fts.h
index cd94956dc55..4c2d247e0a6 100644
--- a/storage/xtradb/include/fts0fts.h
+++ b/storage/xtradb/include/fts0fts.h
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2011, 2018, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2016, 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -705,6 +705,12 @@ fts_drop_index_tables(
 	dict_index_t*	index)			/*!< in: Index to drop */
 	MY_ATTRIBUTE((nonnull, warn_unused_result));
 
+/** Add the table to add to the OPTIMIZER's list.
+@param[in]	table	table to add */
+void
+fts_optimize_add_table(
+	dict_table_t*	table);
+
 /******************************************************************//**
 Remove the table from the OPTIMIZER's list. We do wait for
 acknowledgement from the consumer of the message. */
diff --git a/storage/xtradb/include/fts0priv.h b/storage/xtradb/include/fts0priv.h
index 2d4e9d88fd1..a3936f54a48 100644
--- a/storage/xtradb/include/fts0priv.h
+++ b/storage/xtradb/include/fts0priv.h
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2011, 2018, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -598,22 +598,6 @@ fts_get_table_id(
 					long */
 	MY_ATTRIBUTE((nonnull, warn_unused_result));
 /******************************************************************//**
-Add the table to add to the OPTIMIZER's list. */
-UNIV_INTERN
-void
-fts_optimize_add_table(
-/*===================*/
-	dict_table_t*	table)		/*!< in: table to add */
-	MY_ATTRIBUTE((nonnull));
-/******************************************************************//**
-Optimize a table. */
-UNIV_INTERN
-void
-fts_optimize_do_table(
-/*==================*/
-	dict_table_t*	table)		/*!< in: table to optimize */
-	MY_ATTRIBUTE((nonnull));
-/******************************************************************//**
 Construct the prefix name of an FTS table.
 @return own: table name, must be freed with mem_free() */
 UNIV_INTERN
diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc
index cb496965f01..44e358aab89 100644
--- a/storage/xtradb/row/row0mysql.cc
+++ b/storage/xtradb/row/row0mysql.cc
@@ -3328,6 +3328,10 @@ row_truncate_table_for_mysql(
 		return(DB_TABLESPACE_NOT_FOUND);
 	}
 
+	if (table->fts) {
+		fts_optimize_remove_table(table);
+	}
+
 	trx_start_for_ddl(trx, TRX_DICT_OP_TABLE);
 
 	trx->op_info = "truncating table";
@@ -3734,6 +3738,9 @@ next_rec:
 
 		/* Reset the Doc ID in cache to 0 */
 		if (has_internal_doc_id && table->fts->cache) {
+			DBUG_EXECUTE_IF("ib_trunc_sleep_before_fts_cache_clear",
+					os_thread_sleep(10000000););
+
 			table->fts->fts_status |= TABLE_DICT_LOCKED;
 			fts_update_next_doc_id(trx, table, NULL, 0);
 			fts_cache_clear(table->fts->cache);
@@ -3757,6 +3764,11 @@ funct_exit:
                 table->memcached_sync_count = 0;
         }
 
+	/* Add the table back to FTS optimize background thread. */
+	if (table->fts) {
+		fts_optimize_add_table(table);
+	}
+
 	row_mysql_unlock_data_dictionary(trx);
 
 	dict_stats_update(table, DICT_STATS_EMPTY_TABLE);

From e023f9a4d5a620b54d7f7132567150d80b630692 Mon Sep 17 00:00:00 2001
From: Elena Stepanova <elenst@montyprogram.com>
Date: Tue, 31 Jul 2018 15:19:01 +0300
Subject: [PATCH 084/164] Unstable tests for 10.0.36 release, latest additions

---
 mysql-test/unstable-tests | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests
index 5fa6f6653c3..e8bb02f51e0 100644
--- a/mysql-test/unstable-tests
+++ b/mysql-test/unstable-tests
@@ -23,7 +23,7 @@
 #
 ##############################################################################
 
-# Based on 10.0 5e67567b158db5d294b11b310ff35a454f58d034
+# Based on 10.0 e52315a4a2c7e752e786f9fcf63b0b5685b0d474
 
 main.alter_table                   : Modified in 10.0.36
 main.assign_key_cache              : Added in 10.0.36
@@ -89,12 +89,15 @@ main.statistics                    : Modified in 10.0.36
 main.statistics_close              : Added in 10.0.36
 main.stat_tables                   : Modified in 10.0.36
 main.stat_tables_par_innodb        : MDEV-14155 - wrong rounding
+main.subselect                     : Modified in 10.0.36
 main.subselect_innodb              : MDEV-10614 - sporadic wrong results
 main.subselect_sj                  : Modified in 10.0.36
+main.subselect_sj_mat              : Modified in 10.0.36
 main.subselect_sj2_mat             : Modified in 10.0.36
 main.subselect4                    : Modified in 10.0.36
 main.tc_heuristic_recover          : MDEV-15200 - wrong error on mysqld_stub_cmd
 main.type_datetime                 : MDEV-14322 - wrong result
+main.union                         : Modified in 10.0.36
 main.xa                            : MDEV-11769 - lock wait timeout
 
 #----------------------------------------------------------------
@@ -159,6 +162,7 @@ innodb.innodb_bug48024                       : MDEV-14352 - Assertion failure
 innodb.innodb_bug54044                       : Modified in 10.0.36
 innodb.innodb-mdev7046                       : Modified in 10.0.36
 innodb.innodb_monitor                        : MDEV-10939 - Testcase timeout
+innodb.innodb-wl5522                         : Modified in 10.0.36
 innodb.log_file_size                         : MDEV-15668 - Not found pattern
 innodb.recovery_shutdown                     : MDEV-15671 - Warning: database page corruption
 innodb.rename_table                          : Added in 10.0.36
@@ -168,6 +172,7 @@ innodb.xa_recovery                           : MDEV-15279 - mysqld got exception
 innodb_fts.basic                             : Added in 10.0.36
 innodb_fts.innodb-fts-fic                    : MDEV-14154 - Assertion failure
 innodb_fts.innodb_fts_misc_debug             : MDEV-14156 - Unexpected warning
+innodb_fts.sync_ddl                          : Added in 10.0.36
 
 #----------------------------------------------------------------
 
@@ -214,7 +219,7 @@ perfschema_stress.* : MDEV-10996 - tests not maintained
 #----------------------------------------------------------------
 
 plugins.feedback_plugin_send     : MDEV-7932 - ssl failed for url, MDEV-11118 - wrong result
-plugins.processlist              : MDEV-16818 - Syntax error; added in 10.0.36
+plugins.processlist              : MDEV-16818 - Added in 10.0.36
 plugins.server_audit             : MDEV-9562 - crashes on sol10-sparc; modified in 10.0.36
 plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc, MDEV-14295 - wrong result
 

From f4eac2deeb046a68e4a26ed1a08f8df51bbf817b Mon Sep 17 00:00:00 2001
From: Alexey Botchkov <holyfoot@askmonty.org>
Date: Tue, 31 Jul 2018 16:33:05 +0400
Subject: [PATCH 085/164] MDEV-16054 simple json functions flatline cpu on
 garbage input.

Incorrect char sentence should be handled properly.
---
 mysql-test/r/func_json.result | 6 ++++++
 mysql-test/t/func_json.test   | 7 +++++++
 strings/json_lib.c            | 2 ++
 3 files changed, 15 insertions(+)

diff --git a/mysql-test/r/func_json.result b/mysql-test/r/func_json.result
index 39bd46b7dea..206f745a0d2 100644
--- a/mysql-test/r/func_json.result
+++ b/mysql-test/r/func_json.result
@@ -760,3 +760,9 @@ t1	CREATE TABLE `t1` (
   `c2` varchar(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1
 DROP TABLE t1;
+#
+# MDEV-16054 simple json functions flatline cpu on garbage input.
+#
+select json_array(1,uuid(),compress(5.140264e+307));
+json_array(1,uuid(),compress(5.140264e+307))
+NULL
diff --git a/mysql-test/t/func_json.test b/mysql-test/t/func_json.test
index 37b18763e91..c2cc00dd6e2 100644
--- a/mysql-test/t/func_json.test
+++ b/mysql-test/t/func_json.test
@@ -422,3 +422,10 @@ CREATE TABLE t1 AS SELECT
   JSON_QUOTE(_utf8'foo') AS c2;
 SHOW CREATE TABLE t1;
 DROP TABLE t1;
+
+--echo #
+--echo # MDEV-16054 simple json functions flatline cpu on garbage input.
+--echo #
+
+select json_array(1,uuid(),compress(5.140264e+307));
+
diff --git a/strings/json_lib.c b/strings/json_lib.c
index cbf672f5887..f6b4f15732d 100644
--- a/strings/json_lib.c
+++ b/strings/json_lib.c
@@ -1645,6 +1645,8 @@ int json_escape(CHARSET_INFO *str_cs,
         return -1;
       }
     }
+    else /* c_len == 0, an illegal symbol. */
+      return -1;
   }
 
   return (int)(json - json_start);

From b3e95086e1188dc86e67e44d0fe5a68ea2feca99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Tue, 31 Jul 2018 14:29:05 +0300
Subject: [PATCH 086/164] Fix function pointer type mismatch

---
 storage/innobase/fts/fts0pars.cc | 16 ++++++++--------
 storage/innobase/fts/fts0pars.y  | 16 ++++++++--------
 storage/xtradb/fts/fts0pars.cc   | 16 ++++++++--------
 storage/xtradb/fts/fts0pars.y    | 16 ++++++++--------
 4 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/storage/innobase/fts/fts0pars.cc b/storage/innobase/fts/fts0pars.cc
index 7f0ba4e0c1b..19917ccd26a 100644
--- a/storage/innobase/fts/fts0pars.cc
+++ b/storage/innobase/fts/fts0pars.cc
@@ -87,7 +87,7 @@ extern	int fts_lexer(YYSTYPE*, fts_lexer_t*);
 extern	int fts_blexer(YYSTYPE*, yyscan_t);
 extern	int fts_tlexer(YYSTYPE*, yyscan_t);
 
-typedef int (*fts_scan)();
+
 
 extern int ftserror(const char* p);
 
@@ -102,8 +102,8 @@ extern int ftserror(const char* p);
 
 #define YYTOKENFREE(token) fts_ast_string_free((token))
 
-typedef	int	(*fts_scanner_alt)(YYSTYPE* val, yyscan_t yyscanner);
-typedef	int	(*fts_scanner)();
+
+typedef	int	(*fts_scanner)(YYSTYPE* val, yyscan_t yyscanner);
 
 struct fts_lexer_t {
 	fts_scanner	scanner;
@@ -1950,7 +1950,7 @@ fts_lexer_create(
 			reinterpret_cast<const char*>(query),
 			static_cast<int>(query_len),
 			fts_lexer->yyscanner);
-		fts_lexer->scanner = reinterpret_cast<fts_scan>(fts_blexer);
+		fts_lexer->scanner = fts_blexer;
 		/* FIXME: Debugging */
 		/* fts0bset_debug(1 , fts_lexer->yyscanner); */
 	} else {
@@ -1959,7 +1959,7 @@ fts_lexer_create(
 			reinterpret_cast<const char*>(query),
 			static_cast<int>(query_len),
 			fts_lexer->yyscanner);
-		fts_lexer->scanner = reinterpret_cast<fts_scan>(fts_tlexer);
+		fts_lexer->scanner = fts_tlexer;
 	}
 
 	return(fts_lexer);
@@ -1973,7 +1973,7 @@ fts_lexer_free(
 /*===========*/
 	fts_lexer_t*	fts_lexer)
 {
-	if (fts_lexer->scanner == (fts_scan) fts_blexer) {
+	if (fts_lexer->scanner == fts_blexer) {
 		fts0blex_destroy(fts_lexer->yyscanner);
 	} else {
 		fts0tlex_destroy(fts_lexer->yyscanner);
@@ -1991,9 +1991,9 @@ fts_lexer(
 	YYSTYPE*	val,
 	fts_lexer_t*	fts_lexer)
 {
-	fts_scanner_alt func_ptr;
+	fts_scanner func_ptr;
 
-	func_ptr = (fts_scanner_alt) fts_lexer->scanner;
+	func_ptr = fts_lexer->scanner;
 
 	return(func_ptr(val, fts_lexer->yyscanner));
 }
diff --git a/storage/innobase/fts/fts0pars.y b/storage/innobase/fts/fts0pars.y
index e48036e82fe..65c4189eece 100644
--- a/storage/innobase/fts/fts0pars.y
+++ b/storage/innobase/fts/fts0pars.y
@@ -35,7 +35,7 @@ extern	int fts_lexer(YYSTYPE*, fts_lexer_t*);
 extern	int fts_blexer(YYSTYPE*, yyscan_t);
 extern	int fts_tlexer(YYSTYPE*, yyscan_t);
 
-typedef int (*fts_scan)();
+
 
 extern int ftserror(const char* p);
 
@@ -48,8 +48,8 @@ extern int ftserror(const char* p);
 #define YYPARSE_PARAM state
 #define YYLEX_PARAM ((fts_ast_state_t*) state)->lexer
 
-typedef	int	(*fts_scanner_alt)(YYSTYPE* val, yyscan_t yyscanner);
-typedef	int	(*fts_scanner)();
+
+typedef	int	(*fts_scanner)(YYSTYPE* val, yyscan_t yyscanner);
 
 struct fts_lexer_struct {
 	fts_scanner	scanner;
@@ -238,13 +238,13 @@ fts_lexer_create(
 	if (boolean_mode) {
 		fts0blex_init(&fts_lexer->yyscanner);
 		fts0b_scan_bytes((char*) query, query_len, fts_lexer->yyscanner);
-		fts_lexer->scanner = (fts_scan) fts_blexer;
+		fts_lexer->scanner = fts_blexer;
 		/* FIXME: Debugging */
 		/* fts0bset_debug(1 , fts_lexer->yyscanner); */
 	} else {
 		fts0tlex_init(&fts_lexer->yyscanner);
 		fts0t_scan_bytes((char*) query, query_len, fts_lexer->yyscanner);
-		fts_lexer->scanner = (fts_scan) fts_tlexer;
+		fts_lexer->scanner = fts_tlexer;
 	}
 
 	return(fts_lexer);
@@ -258,7 +258,7 @@ fts_lexer_free(
 /*===========*/
 	fts_lexer_t*	fts_lexer)
 {
-	if (fts_lexer->scanner == (fts_scan) fts_blexer) {
+	if (fts_lexer->scanner == fts_blexer) {
 		fts0blex_destroy(fts_lexer->yyscanner);
 	} else {
 		fts0tlex_destroy(fts_lexer->yyscanner);
@@ -276,9 +276,9 @@ fts_lexer(
 	YYSTYPE*	val,
 	fts_lexer_t*	fts_lexer)
 {
-	fts_scanner_alt func_ptr;
+	fts_scanner func_ptr;
 
-	func_ptr = (fts_scanner_alt) fts_lexer->scanner;
+	func_ptr = fts_lexer->scanner;
 
 	return(func_ptr(val, fts_lexer->yyscanner));
 }
diff --git a/storage/xtradb/fts/fts0pars.cc b/storage/xtradb/fts/fts0pars.cc
index 7f0ba4e0c1b..19917ccd26a 100644
--- a/storage/xtradb/fts/fts0pars.cc
+++ b/storage/xtradb/fts/fts0pars.cc
@@ -87,7 +87,7 @@ extern	int fts_lexer(YYSTYPE*, fts_lexer_t*);
 extern	int fts_blexer(YYSTYPE*, yyscan_t);
 extern	int fts_tlexer(YYSTYPE*, yyscan_t);
 
-typedef int (*fts_scan)();
+
 
 extern int ftserror(const char* p);
 
@@ -102,8 +102,8 @@ extern int ftserror(const char* p);
 
 #define YYTOKENFREE(token) fts_ast_string_free((token))
 
-typedef	int	(*fts_scanner_alt)(YYSTYPE* val, yyscan_t yyscanner);
-typedef	int	(*fts_scanner)();
+
+typedef	int	(*fts_scanner)(YYSTYPE* val, yyscan_t yyscanner);
 
 struct fts_lexer_t {
 	fts_scanner	scanner;
@@ -1950,7 +1950,7 @@ fts_lexer_create(
 			reinterpret_cast<const char*>(query),
 			static_cast<int>(query_len),
 			fts_lexer->yyscanner);
-		fts_lexer->scanner = reinterpret_cast<fts_scan>(fts_blexer);
+		fts_lexer->scanner = fts_blexer;
 		/* FIXME: Debugging */
 		/* fts0bset_debug(1 , fts_lexer->yyscanner); */
 	} else {
@@ -1959,7 +1959,7 @@ fts_lexer_create(
 			reinterpret_cast<const char*>(query),
 			static_cast<int>(query_len),
 			fts_lexer->yyscanner);
-		fts_lexer->scanner = reinterpret_cast<fts_scan>(fts_tlexer);
+		fts_lexer->scanner = fts_tlexer;
 	}
 
 	return(fts_lexer);
@@ -1973,7 +1973,7 @@ fts_lexer_free(
 /*===========*/
 	fts_lexer_t*	fts_lexer)
 {
-	if (fts_lexer->scanner == (fts_scan) fts_blexer) {
+	if (fts_lexer->scanner == fts_blexer) {
 		fts0blex_destroy(fts_lexer->yyscanner);
 	} else {
 		fts0tlex_destroy(fts_lexer->yyscanner);
@@ -1991,9 +1991,9 @@ fts_lexer(
 	YYSTYPE*	val,
 	fts_lexer_t*	fts_lexer)
 {
-	fts_scanner_alt func_ptr;
+	fts_scanner func_ptr;
 
-	func_ptr = (fts_scanner_alt) fts_lexer->scanner;
+	func_ptr = fts_lexer->scanner;
 
 	return(func_ptr(val, fts_lexer->yyscanner));
 }
diff --git a/storage/xtradb/fts/fts0pars.y b/storage/xtradb/fts/fts0pars.y
index e48036e82fe..65c4189eece 100644
--- a/storage/xtradb/fts/fts0pars.y
+++ b/storage/xtradb/fts/fts0pars.y
@@ -35,7 +35,7 @@ extern	int fts_lexer(YYSTYPE*, fts_lexer_t*);
 extern	int fts_blexer(YYSTYPE*, yyscan_t);
 extern	int fts_tlexer(YYSTYPE*, yyscan_t);
 
-typedef int (*fts_scan)();
+
 
 extern int ftserror(const char* p);
 
@@ -48,8 +48,8 @@ extern int ftserror(const char* p);
 #define YYPARSE_PARAM state
 #define YYLEX_PARAM ((fts_ast_state_t*) state)->lexer
 
-typedef	int	(*fts_scanner_alt)(YYSTYPE* val, yyscan_t yyscanner);
-typedef	int	(*fts_scanner)();
+
+typedef	int	(*fts_scanner)(YYSTYPE* val, yyscan_t yyscanner);
 
 struct fts_lexer_struct {
 	fts_scanner	scanner;
@@ -238,13 +238,13 @@ fts_lexer_create(
 	if (boolean_mode) {
 		fts0blex_init(&fts_lexer->yyscanner);
 		fts0b_scan_bytes((char*) query, query_len, fts_lexer->yyscanner);
-		fts_lexer->scanner = (fts_scan) fts_blexer;
+		fts_lexer->scanner = fts_blexer;
 		/* FIXME: Debugging */
 		/* fts0bset_debug(1 , fts_lexer->yyscanner); */
 	} else {
 		fts0tlex_init(&fts_lexer->yyscanner);
 		fts0t_scan_bytes((char*) query, query_len, fts_lexer->yyscanner);
-		fts_lexer->scanner = (fts_scan) fts_tlexer;
+		fts_lexer->scanner = fts_tlexer;
 	}
 
 	return(fts_lexer);
@@ -258,7 +258,7 @@ fts_lexer_free(
 /*===========*/
 	fts_lexer_t*	fts_lexer)
 {
-	if (fts_lexer->scanner == (fts_scan) fts_blexer) {
+	if (fts_lexer->scanner == fts_blexer) {
 		fts0blex_destroy(fts_lexer->yyscanner);
 	} else {
 		fts0tlex_destroy(fts_lexer->yyscanner);
@@ -276,9 +276,9 @@ fts_lexer(
 	YYSTYPE*	val,
 	fts_lexer_t*	fts_lexer)
 {
-	fts_scanner_alt func_ptr;
+	fts_scanner func_ptr;
 
-	func_ptr = (fts_scanner_alt) fts_lexer->scanner;
+	func_ptr = fts_lexer->scanner;
 
 	return(func_ptr(val, fts_lexer->yyscanner));
 }

From a7f84f09bfec22e26b3bd54505a9c8e1d0faff40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Tue, 31 Jul 2018 16:12:38 +0300
Subject: [PATCH 087/164] MDEV-16865 InnoDB fts_query() ignores KILL

The functions fts_ast_visit() and fts_query() inside
InnoDB FULLTEXT INDEX query processing are not checking
for THD::killed (trx_is_interrupted()), like anything
that potentially takes a long time should do.

This is a port of the following change from MySQL 5.7.23,
with a completely rewritten test case.

commit c58c6f8f66ddd0357ecd0c99646aa6bf1dae49c8
Author: Aakanksha Verma <aakanksha.verma@oracle.com>
Date:   Fri May 4 15:53:13 2018 +0530

Bug #27155294 MAX_EXECUTION_TIME NOT INTERUPTED WITH FULLTEXT SEARCH USING MECAB
---
 .../suite/innodb_fts/r/fts_kill_query.result  |  6 ++++
 .../suite/innodb_fts/t/fts_kill_query.test    | 30 +++++++++++++++++++
 storage/innobase/fts/fts0ast.cc               |  9 +++++-
 storage/innobase/fts/fts0que.cc               | 17 ++++++++++-
 storage/innobase/include/fts0ast.h            |  5 +++-
 storage/xtradb/fts/fts0ast.cc                 |  9 +++++-
 storage/xtradb/fts/fts0que.cc                 | 17 ++++++++++-
 storage/xtradb/include/fts0ast.h              |  5 +++-
 8 files changed, 92 insertions(+), 6 deletions(-)
 create mode 100644 mysql-test/suite/innodb_fts/r/fts_kill_query.result
 create mode 100644 mysql-test/suite/innodb_fts/t/fts_kill_query.test

diff --git a/mysql-test/suite/innodb_fts/r/fts_kill_query.result b/mysql-test/suite/innodb_fts/r/fts_kill_query.result
new file mode 100644
index 00000000000..45623f96ab0
--- /dev/null
+++ b/mysql-test/suite/innodb_fts/r/fts_kill_query.result
@@ -0,0 +1,6 @@
+CREATE TABLE t1 (a VARCHAR(7), b text, FULLTEXT KEY idx (a,b)) ENGINE=InnoDB;
+COMMIT;
+SELECT COUNT(*) FROM t1
+WHERE MATCH (a,b) AGAINST ('foo bar' IN BOOLEAN MODE);
+KILL QUERY @id;
+DROP TABLE t1;
diff --git a/mysql-test/suite/innodb_fts/t/fts_kill_query.test b/mysql-test/suite/innodb_fts/t/fts_kill_query.test
new file mode 100644
index 00000000000..3dda29a3876
--- /dev/null
+++ b/mysql-test/suite/innodb_fts/t/fts_kill_query.test
@@ -0,0 +1,30 @@
+--source include/have_innodb.inc
+
+CREATE TABLE t1 (a VARCHAR(7), b text, FULLTEXT KEY idx (a,b)) ENGINE=InnoDB;
+
+--disable_query_log
+BEGIN;
+let $n=1000;
+while ($n) {
+INSERT INTO t1 VALUES('foo bar','boo far');
+dec $n;
+}
+--enable_query_log
+COMMIT;
+
+let $id = `SELECT CONNECTION_ID()`;
+send SELECT COUNT(*) FROM t1
+WHERE MATCH (a,b) AGAINST ('foo bar' IN BOOLEAN MODE);
+
+connect (con1,localhost,root,,);
+let $ignore= `SELECT @id := $ID`;
+KILL QUERY @id;
+disconnect con1;
+
+connection default;
+# The following would return a result set if the KILL was not fast enough.
+--disable_result_log
+--error 0,ER_QUERY_INTERRUPTED,HA_ERR_ABORTED_BY_USER
+reap;
+--enable_result_log
+DROP TABLE t1;
diff --git a/storage/innobase/fts/fts0ast.cc b/storage/innobase/fts/fts0ast.cc
index 030b972440f..4b36152cf62 100644
--- a/storage/innobase/fts/fts0ast.cc
+++ b/storage/innobase/fts/fts0ast.cc
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2014, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2007, 2018, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -27,6 +28,7 @@ Created 2007/3/16 Sunny Bains.
 #include "fts0ast.h"
 #include "fts0pars.h"
 #include "fts0fts.h"
+#include "row0sel.h"
 
 /* The FTS ast visit pass. */
 enum fts_ast_visit_pass_t {
@@ -498,6 +500,7 @@ fts_ast_visit(
 	bool			revisit = false;
 	bool			will_be_ignored = false;
 	fts_ast_visit_pass_t	visit_pass = FTS_PASS_FIRST;
+	const trx_t*		trx = node->trx;
 
 	start_node = node->list.head;
 
@@ -596,6 +599,10 @@ fts_ast_visit(
 		}
 	}
 
+	if (trx_is_interrupted(trx)) {
+		return DB_INTERRUPTED;
+	}
+
 	if (revisit) {
 		/* Exist pass processes the skipped FTS_EXIST operation. */
                 for (node = start_node;
diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc
index 3a543836837..7d4cd7a3646 100644
--- a/storage/innobase/fts/fts0que.cc
+++ b/storage/innobase/fts/fts0que.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2017, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2007, 2018, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -3970,6 +3970,7 @@ fts_query(
 	/* Parse the input query string. */
 	if (fts_query_parse(&query, lc_query_str, result_len)) {
 		fts_ast_node_t*	ast = query.root;
+		ast->trx = trx;
 
 		/* Optimize query to check if it's a single term */
 		fts_query_can_optimize(&query, flags);
@@ -3983,6 +3984,11 @@ fts_query(
 		query.error = fts_ast_visit(
 			FTS_NONE, ast, fts_query_visitor,
 			&query, &will_be_ignored);
+		if (query.error == DB_INTERRUPTED) {
+			error = DB_INTERRUPTED;
+			ut_free(lc_query_str);
+			goto func_exit;
+		}
 
 		/* If query expansion is requested, extend the search
 		with first search pass result */
@@ -4010,6 +4016,15 @@ fts_query(
 		memset(*result, 0, sizeof(**result));
 	}
 
+	if (trx_is_interrupted(trx)) {
+		error = DB_INTERRUPTED;
+		ut_free(lc_query_str);
+		if (result != NULL) {
+			fts_query_free_result(*result);
+		}
+		goto func_exit;
+	}
+
 	ut_free(lc_query_str);
 
 	if (fts_enable_diag_print && (*result)) {
diff --git a/storage/innobase/include/fts0ast.h b/storage/innobase/include/fts0ast.h
index 50f62063893..25df2dd9156 100644
--- a/storage/innobase/include/fts0ast.h
+++ b/storage/innobase/include/fts0ast.h
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2007, 2018, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -314,6 +315,8 @@ struct fts_ast_node_t {
 	fts_ast_node_t*	next_alloc;		/*!< For tracking allocations */
 	bool		visited;		/*!< whether this node is
 						already processed */
+	/** current transaction */
+	const trx_t*	trx;
 };
 
 /* To track state during parsing */
diff --git a/storage/xtradb/fts/fts0ast.cc b/storage/xtradb/fts/fts0ast.cc
index 030b972440f..4b36152cf62 100644
--- a/storage/xtradb/fts/fts0ast.cc
+++ b/storage/xtradb/fts/fts0ast.cc
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2014, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2007, 2018, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -27,6 +28,7 @@ Created 2007/3/16 Sunny Bains.
 #include "fts0ast.h"
 #include "fts0pars.h"
 #include "fts0fts.h"
+#include "row0sel.h"
 
 /* The FTS ast visit pass. */
 enum fts_ast_visit_pass_t {
@@ -498,6 +500,7 @@ fts_ast_visit(
 	bool			revisit = false;
 	bool			will_be_ignored = false;
 	fts_ast_visit_pass_t	visit_pass = FTS_PASS_FIRST;
+	const trx_t*		trx = node->trx;
 
 	start_node = node->list.head;
 
@@ -596,6 +599,10 @@ fts_ast_visit(
 		}
 	}
 
+	if (trx_is_interrupted(trx)) {
+		return DB_INTERRUPTED;
+	}
+
 	if (revisit) {
 		/* Exist pass processes the skipped FTS_EXIST operation. */
                 for (node = start_node;
diff --git a/storage/xtradb/fts/fts0que.cc b/storage/xtradb/fts/fts0que.cc
index 100dbcd70ca..50f198401f9 100644
--- a/storage/xtradb/fts/fts0que.cc
+++ b/storage/xtradb/fts/fts0que.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2017, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2007, 2018, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -3989,6 +3989,7 @@ fts_query(
 	/* Parse the input query string. */
 	if (fts_query_parse(&query, lc_query_str, result_len)) {
 		fts_ast_node_t*	ast = query.root;
+		ast->trx = trx;
 
 		/* Optimize query to check if it's a single term */
 		fts_query_can_optimize(&query, flags);
@@ -4002,6 +4003,11 @@ fts_query(
 		query.error = fts_ast_visit(
 			FTS_NONE, ast, fts_query_visitor,
 			&query, &will_be_ignored);
+		if (query.error == DB_INTERRUPTED) {
+			error = DB_INTERRUPTED;
+			ut_free(lc_query_str);
+			goto func_exit;
+		}
 
 		/* If query expansion is requested, extend the search
 		with first search pass result */
@@ -4029,6 +4035,15 @@ fts_query(
 		memset(*result, 0, sizeof(**result));
 	}
 
+	if (trx_is_interrupted(trx)) {
+		error = DB_INTERRUPTED;
+		ut_free(lc_query_str);
+		if (result != NULL) {
+			fts_query_free_result(*result);
+		}
+		goto func_exit;
+	}
+
 	ut_free(lc_query_str);
 
 	if (fts_enable_diag_print && (*result)) {
diff --git a/storage/xtradb/include/fts0ast.h b/storage/xtradb/include/fts0ast.h
index 50f62063893..25df2dd9156 100644
--- a/storage/xtradb/include/fts0ast.h
+++ b/storage/xtradb/include/fts0ast.h
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2007, 2018, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -314,6 +315,8 @@ struct fts_ast_node_t {
 	fts_ast_node_t*	next_alloc;		/*!< For tracking allocations */
 	bool		visited;		/*!< whether this node is
 						already processed */
+	/** current transaction */
+	const trx_t*	trx;
 };
 
 /* To track state during parsing */

From a90b3862d921ebaeaf3c998cd742e883d2d4af04 Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Wed, 1 Aug 2018 12:41:50 +0100
Subject: [PATCH 088/164] MDEV-16860 MyRocks: support CRC32 instructions on
 Winx64

Compile on Windows MSVC with -DHAVE_SSE2 and -DHAVE_PCLMUL

It is safe, since code will do also runtime checks via cpuid(), before
using the instructions, and will fallback to slower versions,
if instructions are not available.
---
 storage/rocksdb/build_rocksdb.cmake | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/storage/rocksdb/build_rocksdb.cmake b/storage/rocksdb/build_rocksdb.cmake
index 8f01024be63..0f1b77e255d 100644
--- a/storage/rocksdb/build_rocksdb.cmake
+++ b/storage/rocksdb/build_rocksdb.cmake
@@ -376,7 +376,9 @@ SET(SOURCES)
 FOREACH(s ${ROCKSDB_SOURCES})
   list(APPEND SOURCES ${ROCKSDB_SOURCE_DIR}/${s})
 ENDFOREACH()
-
+IF(MSVC)
+  add_definitions(-DHAVE_SSE42 -DHAVE_PCLMUL)
+ENDIF()
 IF(CMAKE_VERSION VERSION_GREATER "2.8.10")
   STRING(TIMESTAMP GIT_DATE_TIME "%Y-%m-%d %H:%M:%S")
 ENDIF()

From 90b66c169957962de612bc951cdf369cdfccc99c Mon Sep 17 00:00:00 2001
From: Daniel Bartholomew <db@dbart.us>
Date: Wed, 1 Aug 2018 12:09:33 -0400
Subject: [PATCH 089/164] bump the VERSION

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 59a439d4152..2eee908ea01 100644
--- a/VERSION
+++ b/VERSION
@@ -1,3 +1,3 @@
 MYSQL_VERSION_MAJOR=10
 MYSQL_VERSION_MINOR=0
-MYSQL_VERSION_PATCH=36
+MYSQL_VERSION_PATCH=37

From 1b87cd80a268d333acbf4fc4a3d7527cb5da3d4d Mon Sep 17 00:00:00 2001
From: Alexander Barkov <bar@mariadb.com>
Date: Thu, 2 Aug 2018 10:48:55 +0400
Subject: [PATCH 090/164] MDEV-16878 Functions ADDTIME and SUBTIME get wrongly
 removed from WHERE by the equal expression optimizer

---
 mysql-test/main/func_time.result | 22 ++++++++++++++++++++++
 mysql-test/main/func_time.test   | 14 ++++++++++++++
 mysql-test/main/gis.result       | 12 ++++++------
 sql/item_timefunc.cc             | 21 ---------------------
 sql/item_timefunc.h              |  6 ++++--
 5 files changed, 46 insertions(+), 29 deletions(-)

diff --git a/mysql-test/main/func_time.result b/mysql-test/main/func_time.result
index 151dbc8d5d8..cda36932be0 100644
--- a/mysql-test/main/func_time.result
+++ b/mysql-test/main/func_time.result
@@ -3537,3 +3537,25 @@ SET @@session.slow_query_log= @sav_slow_query_log;
 DROP FUNCTION fn_sleep_before_now;
 DROP TRIGGER trg_insert_t_ts;
 DROP TABLE t_ts, t_trig;
+#
+# MDEV-16878 Functions ADDTIME and SUBTIME get wrongly removed from WHERE by the equal expression optimizer
+#
+CREATE TABLE t1 (a TIME);
+INSERT INTO t1 VALUES ('00:00:10'),('00:00:20');
+SELECT a, SUBTIME(a,TIME'00:00:01'), ADDTIME(a,TIME'00:00:01') FROM t1;
+a	SUBTIME(a,TIME'00:00:01')	ADDTIME(a,TIME'00:00:01')
+00:00:10	00:00:09	00:00:11
+00:00:20	00:00:19	00:00:21
+SELECT * FROM t1 WHERE SUBTIME(a,TIME'00:00:01')=TIME'00:00:09';
+a
+00:00:10
+SELECT * FROM t1 WHERE ADDTIME(a,TIME'00:00:01')<=TIME'00:00:09';
+a
+SELECT * FROM t1 WHERE SUBTIME(a,TIME'00:00:01')=TIME'00:00:09' AND ADDTIME(a,TIME'00:00:01')<=TIME'00:00:09';
+a
+EXPLAIN EXTENDED SELECT * FROM t1 WHERE SUBTIME(a,TIME'00:00:01')=TIME'00:00:09' AND ADDTIME(a,TIME'00:00:01')<=TIME'00:00:09';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	2	100.00	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where subtime(`test`.`t1`.`a`,TIME'00:00:01') = TIME'00:00:09' and addtime(`test`.`t1`.`a`,TIME'00:00:01') <= TIME'00:00:09'
+DROP TABLE t1;
diff --git a/mysql-test/main/func_time.test b/mysql-test/main/func_time.test
index ecbf23f8e36..cbce3dc26bc 100644
--- a/mysql-test/main/func_time.test
+++ b/mysql-test/main/func_time.test
@@ -2135,3 +2135,17 @@ DROP TABLE t_ts, t_trig;
 #
 # End of MDEV-13727
 ###################
+
+
+--echo #
+--echo # MDEV-16878 Functions ADDTIME and SUBTIME get wrongly removed from WHERE by the equal expression optimizer
+--echo #
+
+CREATE TABLE t1 (a TIME);
+INSERT INTO t1 VALUES ('00:00:10'),('00:00:20');
+SELECT a, SUBTIME(a,TIME'00:00:01'), ADDTIME(a,TIME'00:00:01') FROM t1;
+SELECT * FROM t1 WHERE SUBTIME(a,TIME'00:00:01')=TIME'00:00:09';
+SELECT * FROM t1 WHERE ADDTIME(a,TIME'00:00:01')<=TIME'00:00:09'; 
+SELECT * FROM t1 WHERE SUBTIME(a,TIME'00:00:01')=TIME'00:00:09' AND ADDTIME(a,TIME'00:00:01')<=TIME'00:00:09'; 
+EXPLAIN EXTENDED SELECT * FROM t1 WHERE SUBTIME(a,TIME'00:00:01')=TIME'00:00:09' AND ADDTIME(a,TIME'00:00:01')<=TIME'00:00:09'; 
+DROP TABLE t1;
diff --git a/mysql-test/main/gis.result b/mysql-test/main/gis.result
index 3f2e6d39db8..2fb2d32f447 100644
--- a/mysql-test/main/gis.result
+++ b/mysql-test/main/gis.result
@@ -4128,17 +4128,17 @@ ERROR HY000: Illegal parameter data types geometry and interval for operation 'd
 SELECT INTERVAL 10 DAY + POINT(1,1);
 ERROR HY000: Illegal parameter data types geometry and interval for operation 'date_add_interval'
 SELECT ADDTIME(POINT(1,1), '10:10:10');
-ERROR HY000: Illegal parameter data types geometry and varchar for operation 'add_time'
+ERROR HY000: Illegal parameter data types geometry and varchar for operation 'addtime'
 SELECT ADDTIME('10:10:10', POINT(1,1));
-ERROR HY000: Illegal parameter data types varchar and geometry for operation 'add_time'
+ERROR HY000: Illegal parameter data types varchar and geometry for operation 'addtime'
 SELECT ADDTIME(POINT(1,1), TIME'10:10:10');
-ERROR HY000: Illegal parameter data types geometry and time for operation 'add_time'
+ERROR HY000: Illegal parameter data types geometry and time for operation 'addtime'
 SELECT ADDTIME(TIME'10:10:10', POINT(1,1));
-ERROR HY000: Illegal parameter data types time and geometry for operation 'add_time'
+ERROR HY000: Illegal parameter data types time and geometry for operation 'addtime'
 SELECT ADDTIME(POINT(1,1), TIMESTAMP'2001-01-01 10:10:10');
-ERROR HY000: Illegal parameter data types geometry and datetime for operation 'add_time'
+ERROR HY000: Illegal parameter data types geometry and datetime for operation 'addtime'
 SELECT ADDTIME(TIMESTAMP'2001-01-01 10:10:10', POINT(1,1));
-ERROR HY000: Illegal parameter data types datetime and geometry for operation 'add_time'
+ERROR HY000: Illegal parameter data types datetime and geometry for operation 'addtime'
 SELECT STR_TO_DATE(POINT(1,1),'%M %d,%Y');
 ERROR HY000: Illegal parameter data types geometry and varchar for operation 'str_to_date'
 SELECT STR_TO_DATE('2001-01-01', POINT(1,1));
diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc
index b4d929e0f70..d29181a3446 100644
--- a/sql/item_timefunc.cc
+++ b/sql/item_timefunc.cc
@@ -2817,27 +2817,6 @@ bool Item_func_add_time::get_date(MYSQL_TIME *ltime, ulonglong fuzzy_date)
 }
 
 
-void Item_func_add_time::print(String *str, enum_query_type query_type)
-{
-  if (is_date)
-  {
-    DBUG_ASSERT(sign > 0);
-    str->append(STRING_WITH_LEN("timestamp("));
-  }
-  else
-  {
-    if (sign > 0)
-      str->append(STRING_WITH_LEN("addtime("));
-    else
-      str->append(STRING_WITH_LEN("subtime("));
-  }
-  args[0]->print(str, query_type);
-  str->append(',');
-  args[1]->print(str, query_type);
-  str->append(')');
-}
-
-
 /**
   TIMEDIFF(t,s) is a time function that calculates the 
   time value between a start and end time.
diff --git a/sql/item_timefunc.h b/sql/item_timefunc.h
index 51ce3bf2988..7aacdec85e0 100644
--- a/sql/item_timefunc.h
+++ b/sql/item_timefunc.h
@@ -1245,8 +1245,10 @@ public:
   { sign= neg_arg ? -1 : 1; }
   bool fix_length_and_dec();
   bool get_date(MYSQL_TIME *ltime, ulonglong fuzzy_date);
-  void print(String *str, enum_query_type query_type);
-  const char *func_name() const { return "add_time"; }
+  const char *func_name() const
+  {
+    return is_date ? "timestamp" : sign > 0 ? "addtime" : "subtime";
+  }
   Item *get_copy(THD *thd)
   { return get_item_copy<Item_func_add_time>(thd, this); }
 };

From a89a6e4f774d9b01bee0d33c0b2f0d6a1dfae39d Mon Sep 17 00:00:00 2001
From: Rasmus Johansson <rasmus@mariadb.com>
Date: Thu, 2 Aug 2018 11:27:22 +0000
Subject: [PATCH 091/164] Added -j option to dpkg-buildpackage to speed up
 build

---
 debian/autobake-deb.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh
index 56575211c61..4d488767a97 100755
--- a/debian/autobake-deb.sh
+++ b/debian/autobake-deb.sh
@@ -160,7 +160,7 @@ fi
 # Build the package
 # Pass -I so that .git and other unnecessary temporary and source control files
 # will be ignored by dpkg-source when creating the tar.gz source package.
-fakeroot dpkg-buildpackage -us -uc -I $BUILDPACKAGE_FLAGS
+fakeroot dpkg-buildpackage -us -uc -I $BUILDPACKAGE_FLAGS -j$(nproc)
 
 # If the step above fails due to missing dependencies, you can manually run
 #   sudo mk-build-deps debian/control -r -i

From 89b6ce026a2bbbbd118df19da9ae9a81aabe4f65 Mon Sep 17 00:00:00 2001
From: Michael Widenius <monty@mariadb.org>
Date: Tue, 31 Jul 2018 15:47:54 +0300
Subject: [PATCH 092/164] Disabled rocksdb.autoinc_debug as it fails very often

---
 storage/rocksdb/mysql-test/rocksdb/t/disabled.def | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def
index 3b9726986f0..d0cd1ce6407 100644
--- a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def
+++ b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def
@@ -76,4 +76,4 @@ information_schema: MDEV-14372: unstable testcase
 ##
 
 mysqlbinlog_gtid_skip_empty_trans_rocksdb : MariaRocks: requires GTIDs
-
+autoinc_debug: Fails with wrong results

From b4f7f12e2ba2d805f9e8c424d2082beacca0111d Mon Sep 17 00:00:00 2001
From: Oleksandr Byelkin <sanja@mariadb.com>
Date: Thu, 2 Aug 2018 20:27:10 +0200
Subject: [PATCH 093/164] fix galera test.

---
 .../r/galera_sst_mysqldump_with_key.result    | 97 +++++++++++++++++++
 1 file changed, 97 insertions(+)

diff --git a/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result b/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result
index bdf3844f2c5..bf9613f6d7d 100644
--- a/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result
+++ b/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result
@@ -183,6 +183,103 @@ COUNT(*) = 0
 DROP TABLE t1;
 COMMIT;
 SET AUTOCOMMIT=ON;
+Performing State Transfer on a server that has been killed and restarted
+while a DDL was in progress on it
+CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+COMMIT;
+SET GLOBAL debug_dbug = 'd,sync.alter_opened_table';
+ALTER TABLE t1 ADD COLUMN f2 INTEGER;
+SET wsrep_sync_wait = 0;
+Killing server ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 (f1) VALUES ('node1_committed_during');
+INSERT INTO t1 (f1) VALUES ('node1_committed_during');
+INSERT INTO t1 (f1) VALUES ('node1_committed_during');
+INSERT INTO t1 (f1) VALUES ('node1_committed_during');
+INSERT INTO t1 (f1) VALUES ('node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+Performing --wsrep-recover ...
+Starting server ...
+Using --wsrep-start-position when starting mysqld ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 (f1) VALUES ('node2_committed_after');
+INSERT INTO t1 (f1) VALUES ('node2_committed_after');
+INSERT INTO t1 (f1) VALUES ('node2_committed_after');
+INSERT INTO t1 (f1) VALUES ('node2_committed_after');
+INSERT INTO t1 (f1) VALUES ('node2_committed_after');
+COMMIT;
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 (f1) VALUES ('node1_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_committed_after');
+COMMIT;
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+ROLLBACK;
+SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
+COUNT(*) = 2
+1
+SELECT COUNT(*) = 35 FROM t1;
+COUNT(*) = 35
+1
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+SET AUTOCOMMIT=ON;
+SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
+COUNT(*) = 2
+1
+SELECT COUNT(*) = 35 FROM t1;
+COUNT(*) = 35
+1
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+SET AUTOCOMMIT=ON;
+SET GLOBAL debug_dbug = $debug_orig;
 CALL mtr.add_suppression("Slave SQL: Error 'The MySQL server is running with the --skip-grant-tables option so it cannot execute this statement' on query");
 DROP USER sst;
 CALL mtr.add_suppression("Slave SQL: Error 'The MySQL server is running with the --skip-grant-tables option so it cannot execute this statement' on query");

From 8ae2a2dbe6fc52d40ec88c325b9a44de52f83f2f Mon Sep 17 00:00:00 2001
From: Alexander Barkov <bar@mariadb.com>
Date: Fri, 3 Aug 2018 09:03:11 +0400
Subject: [PATCH 094/164] MDEV-15363 Wrong result for
 CAST(LAST_DAY(TIME'00:00:00') AS TIME)

This problem was earlier fixed by the patch for MDEV-15340.
Adding tests only.
---
 mysql-test/main/func_time.result |  9 +++++++++
 mysql-test/main/func_time.test   | 10 ++++++++++
 2 files changed, 19 insertions(+)

diff --git a/mysql-test/main/func_time.result b/mysql-test/main/func_time.result
index cda36932be0..8a7a990bb85 100644
--- a/mysql-test/main/func_time.result
+++ b/mysql-test/main/func_time.result
@@ -3559,3 +3559,12 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 Warnings:
 Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where subtime(`test`.`t1`.`a`,TIME'00:00:01') = TIME'00:00:09' and addtime(`test`.`t1`.`a`,TIME'00:00:01') <= TIME'00:00:09'
 DROP TABLE t1;
+#
+# MDEV-15363 Wrong result for CAST(LAST_DAY(TIME'00:00:00') AS TIME)
+#
+SELECT
+LAST_DAY(TIME'00:00:00') AS c1,
+CAST(CAST(LAST_DAY(TIME'00:00:00') AS DATE) AS TIME) AS c2,
+CAST(LAST_DAY(TIME'00:00:00') AS TIME) AS c3;
+c1	c2	c3
+2018-08-31	00:00:00	00:00:00
diff --git a/mysql-test/main/func_time.test b/mysql-test/main/func_time.test
index cbce3dc26bc..53cfd29dc9c 100644
--- a/mysql-test/main/func_time.test
+++ b/mysql-test/main/func_time.test
@@ -2149,3 +2149,13 @@ SELECT * FROM t1 WHERE ADDTIME(a,TIME'00:00:01')<=TIME'00:00:09';
 SELECT * FROM t1 WHERE SUBTIME(a,TIME'00:00:01')=TIME'00:00:09' AND ADDTIME(a,TIME'00:00:01')<=TIME'00:00:09'; 
 EXPLAIN EXTENDED SELECT * FROM t1 WHERE SUBTIME(a,TIME'00:00:01')=TIME'00:00:09' AND ADDTIME(a,TIME'00:00:01')<=TIME'00:00:09'; 
 DROP TABLE t1;
+
+
+--echo #
+--echo # MDEV-15363 Wrong result for CAST(LAST_DAY(TIME'00:00:00') AS TIME)
+--echo #
+
+SELECT
+  LAST_DAY(TIME'00:00:00') AS c1,
+  CAST(CAST(LAST_DAY(TIME'00:00:00') AS DATE) AS TIME) AS c2,
+  CAST(LAST_DAY(TIME'00:00:00') AS TIME) AS c3;

From f867a695f806308173bfd579e972270a554f87af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Thu, 2 Aug 2018 18:09:37 +0300
Subject: [PATCH 095/164] After-merge fix: Remove an unnecessary parameter

---
 storage/innobase/handler/ha_innodb.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index d8e805862f7..dccdeacb08f 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -13534,7 +13534,6 @@ inline MY_ATTRIBUTE((warn_unused_result))
 dberr_t
 innobase_rename_table(
 /*==================*/
-	THD*            thd,    /*!< Connection thread handle */
 	trx_t*		trx,	/*!< in: transaction */
 	const char*	from,	/*!< in: old name of the table */
 	const char*	to)	/*!< in: new name of the table */
@@ -13565,7 +13564,7 @@ innobase_rename_table(
 
 	/* Since DICT_BG_YIELD has sleep for 250 milliseconds,
 	Convert lock_wait_timeout unit from second to 250 milliseconds */
-	long int lock_wait_timeout = thd_lock_wait_timeout(thd) * 4;
+	long int lock_wait_timeout = thd_lock_wait_timeout(trx->mysql_thd) * 4;
 	if (table != NULL) {
 		for (dict_index_t* index = dict_table_get_first_index(table);
 		     index != NULL;
@@ -13680,7 +13679,7 @@ ha_innobase::rename_table(
 	++trx->will_lock;
 	trx_set_dict_operation(trx, TRX_DICT_OP_INDEX);
 
-	dberr_t	error = innobase_rename_table(thd, trx, from, to);
+	dberr_t	error = innobase_rename_table(trx, from, to);
 
 	DEBUG_SYNC(thd, "after_innobase_rename_table");
 

From a1b23361990487c9f125e4f922331f01605ca9ec Mon Sep 17 00:00:00 2001
From: Sergey Vojtovich <svoj@mariadb.org>
Date: Thu, 30 Nov 2017 18:07:28 +0400
Subject: [PATCH 096/164] Optimized away excessive condition

trx_set_rw_mode() is never called for read-only transactions, this is guarded
by callers.

Removing this condition from critical section immediately gives 5% scalability
improvement in OLTP index updates benchmark.
---
 storage/innobase/trx/trx0trx.cc | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc
index deacc244154..7cf31ef89eb 100644
--- a/storage/innobase/trx/trx0trx.cc
+++ b/storage/innobase/trx/trx0trx.cc
@@ -2957,6 +2957,7 @@ trx_set_rw_mode(
 	ut_ad(trx->rsegs.m_redo.rseg == 0);
 	ut_ad(!trx->in_rw_trx_list);
 	ut_ad(!trx_is_autocommit_non_locking(trx));
+	ut_ad(!trx->read_only);
 
 	if (high_level_read_only) {
 		return;
@@ -2993,11 +2994,9 @@ trx_set_rw_mode(
 	}
 #endif /* UNIV_DEBUG */
 
-	if (!trx->read_only) {
-		UT_LIST_ADD_FIRST(trx_sys->rw_trx_list, trx);
+	UT_LIST_ADD_FIRST(trx_sys->rw_trx_list, trx);
 
-		ut_d(trx->in_rw_trx_list = true);
-	}
+	ut_d(trx->in_rw_trx_list = true);
 
 	mutex_exit(&trx_sys->mutex);
 }

From f70a3185765fb623668e927122484da24341a85f Mon Sep 17 00:00:00 2001
From: Allen Lai <zheng.lai@oracle.com>
Date: Sun, 8 Apr 2018 09:16:56 +0800
Subject: [PATCH 097/164] Bug#27805553 HARD ERROR SHOULD BE REPORTED WHEN
 FSYNC() RETURN EIO.

fsync() will just return EIO only once when the IO error happens, so, it's
wrong to keep trying to call it till it return success.
When fsync() returns EIO it should be treated as a hard error and InnoDB must
abort immediately.
---
 storage/innobase/os/os0file.cc | 21 +++------------------
 1 file changed, 3 insertions(+), 18 deletions(-)

diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc
index 9853263600e..ad8a76c5280 100644
--- a/storage/innobase/os/os0file.cc
+++ b/storage/innobase/os/os0file.cc
@@ -2493,30 +2493,15 @@ os_file_fsync_posix(
 			os_thread_sleep(200000);
 			break;
 
-		case EIO:
-
-			++failures;
-			ut_a(failures < 1000);
-
-			if (!(failures % 100)) {
-
-				ib::warn()
-					<< "fsync(): "
-					<< "An error occurred during "
-					<< "synchronization,"
-					<< " retrying";
-			}
-
-			/* 0.2 sec */
-			os_thread_sleep(200000);
-			break;
-
 		case EINTR:
 
 			++failures;
 			ut_a(failures < 2000);
 			break;
 
+		case EIO:
+			ib::error() << "fsync() returned EIO, aborting";
+			/* fall through */
 		default:
 			ut_error;
 			break;

From 204608974536a99e0c218b909b7137395ab8cd22 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Tue, 31 Jul 2018 18:35:13 +0300
Subject: [PATCH 098/164] MDEV-14637: Fix hang due to DDL with FOREIGN KEY

When MySQL 5.7.1 introduced WL#6326 to reduce contention on the
non-leaf levels of B-trees, it introduced a new rw-lock mode SX
(not conflicting with S, but conflicting with SX and X) and
new rules to go with it.

A thread that is holding an dict_index_t::lock aka index->lock
in SX mode is permitted to acquire non-leaf buf_block_t::lock
aka block->lock X or SX mode, in monotonically descending order.
That is, once the thread has acquired a block->lock, it is not
allowed to acquire a lock on its parent or grandparent pages.
Such arbitrary-order access is only allowed when the thread
acquired the index->lock in X mode upfront.

A customer encountered a repeatable hang when loading a dump into
InnoDB while using multiple innodb_purge_threads (default: 4).
The dump makes very heavy use of FOREIGN KEY constraints.
By luck, it happened so that two purge worker threads (srv_worker_thread)
deadlocked with each other. Both were operating on the index FOR_REF
of the InnoDB internal table SYS_FOREIGN. One of them was legitimately
holding index->lock S-latch and the root block->lock S-latch. The other
had acquired index->lock SX-latch, root block->lock SX-latch, and a bunch
of other latches, including the fil_space_t::latch for freeing some blocks
and some leaf page latches. This other thread was inside 2 nested calls
to btr_compress() and it was trying to reacquire the root block->lock
in X mode, violating the WL#6326 protocol.

This violation led to a deadlock, because while S is compatible with SX
and a thread can upgrade an SX lock to X when there are no conflicting
requests, in this case there was a conflicting S lock held by the other
purge worker thread.

During this deadlock, both threads are holding dict_operation_lock S-latch,
which would block any subsequent DDL statements, such as CREATE TABLE.

The tables SYS_FOREIGN and SYS_FOREIGN_COLS are special in that they
define key columns of the type VARCHAR(0), created using the InnoDB
internal SQL parser. Because InnoDB does not internally enforce the
maximum length of columns, it would happily write more than 0 bytes
to these columns. This caused a miscalculation of node_ptr_max_size.

btr_cur_will_modify_tree(): Clean up some code. (No functional change.)

btr_node_ptr_max_size(): Renamed from dict_index_node_ptr_max_size().
Use a more realistic maximum size for SYS_FOREIGN and SYS_FOREIGN_COLS.

btr_cur_pessimistic_delete(): Refrain from merging pages if it is
not safe.

This work is based on the following MySQL 5.7.23 fix:

commit 58dcf0b4a4165ed59de94a9a1e7d8c954f733726
Author: Aakanksha Verma <aakanksha.verma@oracle.com>
Date:   Wed May 9 18:54:03 2018 +0530

    BUG#26225783 MYSQL CRASH ON CREATE TABLE (REPRODUCEABLE) -> INNODB: A
    LONG SEMAPHORE WAIT
---
 storage/innobase/btr/btr0cur.cc      | 160 +++++++++++++++++++++++----
 storage/innobase/dict/dict0dict.cc   |  87 ---------------
 storage/innobase/include/dict0dict.h |   8 --
 3 files changed, 136 insertions(+), 119 deletions(-)

diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc
index af095ade945..cb03562b598 100644
--- a/storage/innobase/btr/btr0cur.cc
+++ b/storage/innobase/btr/btr0cur.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1994, 2018, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2008, Google Inc.
 Copyright (c) 2012, Facebook Inc.
 Copyright (c) 2015, 2018, MariaDB Corporation.
@@ -586,13 +586,15 @@ btr_cur_will_modify_tree(
 	first record and following compress might delete the record and causes
 	the uppper level node_ptr modification. */
 
+	const ulint n_recs = page_get_n_recs(page);
+
 	if (lock_intention <= BTR_INTENTION_BOTH) {
 		ulint	margin;
 
 		/* check delete will cause. (BTR_INTENTION_BOTH
 		or BTR_INTENTION_DELETE) */
 		/* first, 2nd, 2nd-last and last records are 4 records */
-		if (page_get_n_recs(page) < 5) {
+		if (n_recs < 5) {
 			return(true);
 		}
 
@@ -638,8 +640,7 @@ btr_cur_will_modify_tree(
 		/* Once we invoke the btr_cur_limit_optimistic_insert_debug,
 		we should check it here in advance, since the max allowable
 		records in a page is limited. */
-		LIMIT_OPTIMISTIC_INSERT_DEBUG(page_get_n_recs(page),
-					      return(true));
+		LIMIT_OPTIMISTIC_INSERT_DEBUG(n_recs, return true);
 
 		/* needs 2 records' space for the case the single split and
 		insert cannot fit.
@@ -652,18 +653,16 @@ btr_cur_will_modify_tree(
 		    || max_size < rec_size * 2) {
 			return(true);
 		}
-		/* TODO: optimize this condition for compressed page.
-		this is based on the worst compress rate.
-		currently looking only uncompressed page, but we can look
-		also compressed page page_zip_available() if already in the
-		buffer pool */
+
+		/* TODO: optimize this condition for ROW_FORMAT=COMPRESSED.
+		This is based on the worst case, and we could invoke
+		page_zip_available() on the block->page.zip. */
 		/* needs 2 records' space also for worst compress rate. */
 		if (page_size.is_compressed()
 		    && page_zip_empty_size(index->n_fields,
 					   page_size.physical())
-		       < rec_size * 2 + page_get_data_size(page)
-			 + page_dir_calc_reserved_space(
-				page_get_n_recs(page) + 2) + 1) {
+		    <= rec_size * 2 + page_get_data_size(page)
+		    + page_dir_calc_reserved_space(n_recs + 2)) {
 			return(true);
 		}
 	}
@@ -701,6 +700,98 @@ btr_cur_need_opposite_intention(
 	return(false);
 }
 
+/**
+@param[in]	index b-tree
+@return maximum size of a node pointer record in bytes */
+static ulint btr_node_ptr_max_size(const dict_index_t* index)
+{
+	if (dict_index_is_ibuf(index)) {
+		/* cannot estimate accurately */
+		/* This is universal index for change buffer.
+		The max size of the entry is about max key length * 2.
+		(index key + primary key to be inserted to the index)
+		(The max key length is UNIV_PAGE_SIZE / 16 * 3 at
+		 ha_innobase::max_supported_key_length(),
+		 considering MAX_KEY_LENGTH = 3072 at MySQL imposes
+		 the 3500 historical InnoDB value for 16K page size case.)
+		For the universal index, node_ptr contains most of the entry.
+		And 512 is enough to contain ibuf columns and meta-data */
+		return srv_page_size / 8 * 3 + 512;
+	}
+
+	/* Each record has page_no, length of page_no and header. */
+	ulint comp = dict_table_is_comp(index->table);
+	ulint rec_max_size = comp
+		? REC_NODE_PTR_SIZE + 1 + REC_N_NEW_EXTRA_BYTES
+		+ UT_BITS_IN_BYTES(index->n_nullable)
+		: REC_NODE_PTR_SIZE + 2 + REC_N_OLD_EXTRA_BYTES
+		+ 2 * index->n_fields;
+
+	/* Compute the maximum possible record size. */
+	for (ulint i = 0; i < dict_index_get_n_unique_in_tree(index); i++) {
+		const dict_field_t*	field
+			= dict_index_get_nth_field(index, i);
+		const dict_col_t*	col
+			= dict_field_get_col(field);
+		ulint			field_max_size;
+		ulint			field_ext_max_size;
+
+		/* Determine the maximum length of the index field. */
+
+		field_max_size = dict_col_get_fixed_size(col, comp);
+		if (field_max_size) {
+			/* dict_index_add_col() should guarantee this */
+			ut_ad(!field->prefix_len
+			      || field->fixed_len == field->prefix_len);
+			/* Fixed lengths are not encoded
+			in ROW_FORMAT=COMPACT. */
+			rec_max_size += field_max_size;
+			continue;
+		}
+
+		field_max_size = dict_col_get_max_size(col);
+		if (UNIV_UNLIKELY(!field_max_size)) {
+			/* SYS_FOREIGN.ID is defined as CHAR in the
+			InnoDB internal SQL parser, which translates
+			into the incorrect VARCHAR(0).  InnoDB does
+			not enforce maximum lengths of columns, so
+			that is why any data can be inserted in the
+			first place.
+
+			Likewise, SYS_FOREIGN.FOR_NAME,
+			SYS_FOREIGN.REF_NAME, SYS_FOREIGN_COLS.ID, are
+			defined as CHAR, and also they are part of a key. */
+
+			ut_ad(!strcmp(index->table->name.m_name,
+				      "SYS_FOREIGN")
+			      || !strcmp(index->table->name.m_name,
+					 "SYS_FOREIGN_COLS"));
+			ut_ad(!comp);
+
+			rec_max_size += (srv_page_size == UNIV_PAGE_SIZE_MAX)
+				? REDUNDANT_REC_MAX_DATA_SIZE
+				: page_get_free_space_of_empty(FALSE) / 2;
+		}
+		field_ext_max_size = field_max_size < 256 ? 1 : 2;
+
+		if (field->prefix_len
+		    && field->prefix_len < field_max_size) {
+			field_max_size = field->prefix_len;
+		}
+
+		if (comp) {
+			/* Add the extra size for ROW_FORMAT=COMPACT.
+			For ROW_FORMAT=REDUNDANT, these bytes were
+			added to rec_max_size before this loop. */
+			rec_max_size += field_ext_max_size;
+		}
+
+		rec_max_size += field_max_size;
+	}
+
+	return rec_max_size;
+}
+
 /********************************************************************//**
 Searches an index tree and positions a tree cursor on a given level.
 NOTE: n_fields_cmp in tuple must be set so that it cannot be compared
@@ -1028,7 +1119,7 @@ btr_cur_search_to_nth_level(
 	page_id_t		page_id(space, dict_index_get_page(index));
 
 	if (root_leaf_rw_latch == RW_X_LATCH) {
-		node_ptr_max_size = dict_index_node_ptr_max_size(index);
+		node_ptr_max_size = btr_node_ptr_max_size(index);
 	}
 
 	up_match = 0;
@@ -2128,7 +2219,7 @@ btr_cur_open_at_index_side_func(
 	const page_size_t&	page_size = dict_table_page_size(index->table);
 
 	if (root_leaf_rw_latch == RW_X_LATCH) {
-		node_ptr_max_size = dict_index_node_ptr_max_size(index);
+		node_ptr_max_size = btr_node_ptr_max_size(index);
 	}
 
 	height = ULINT_UNDEFINED;
@@ -2487,7 +2578,7 @@ btr_cur_open_at_rnd_pos_func(
 	dberr_t			err = DB_SUCCESS;
 
 	if (root_leaf_rw_latch == RW_X_LATCH) {
-		node_ptr_max_size = dict_index_node_ptr_max_size(index);
+		node_ptr_max_size = btr_node_ptr_max_size(index);
 	}
 
 	height = ULINT_UNDEFINED;
@@ -5162,7 +5253,6 @@ btr_cur_pessimistic_delete(
 		btr_discard_page(cursor, mtr);
 
 		ret = TRUE;
-
 		goto return_after_reservations;
 	}
 
@@ -5236,23 +5326,45 @@ btr_cur_pessimistic_delete(
 		}
 	}
 
-	page_cur_delete_rec(btr_cur_get_page_cur(cursor), index, offsets, mtr);
+	/* SPATIAL INDEX never use SX locks; we can allow page merges
+	while holding X lock on the spatial index tree.
+	Do not allow merges of non-leaf B-tree pages unless it is
+	safe to do so. */
+	{
+		const bool allow_merge = page_is_leaf(page)
+			|| dict_index_is_spatial(index)
+			|| btr_cur_will_modify_tree(
+				index, page, BTR_INTENTION_DELETE, rec,
+				btr_node_ptr_max_size(index),
+				block->page.size, mtr);
+		page_cur_delete_rec(btr_cur_get_page_cur(cursor), index,
+				    offsets, mtr);
 #ifdef UNIV_ZIP_DEBUG
-	ut_a(!page_zip || page_zip_validate(page_zip, page, index));
+		ut_a(!page_zip || page_zip_validate(page_zip, page, index));
 #endif /* UNIV_ZIP_DEBUG */
 
-	/* btr_check_node_ptr() needs parent block latched */
-	ut_ad(!parent_latched || btr_check_node_ptr(index, block, mtr));
+		ut_ad(!parent_latched
+		      || btr_check_node_ptr(index, block, mtr));
+
+		if (!ret && btr_cur_compress_recommendation(cursor, mtr)) {
+			if (UNIV_LIKELY(allow_merge)) {
+				ret = btr_cur_compress_if_useful(
+					cursor, FALSE, mtr);
+			} else {
+				ib::warn() << "Not merging page "
+					   << block->page.id
+					   << " in index " << index->name
+					   << " of " << index->table->name;
+				ut_ad(!"MDEV-14637");
+			}
+		}
+	}
 
 return_after_reservations:
 	*err = DB_SUCCESS;
 
 	mem_heap_free(heap);
 
-	if (ret == FALSE) {
-		ret = btr_cur_compress_if_useful(cursor, FALSE, mtr);
-	}
-
 	if (!srv_read_only_mode
 	    && page_is_leaf(page)
 	    && !dict_index_is_online_ddl(index)) {
diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc
index 8dc29863367..a0d430e0257 100644
--- a/storage/innobase/dict/dict0dict.cc
+++ b/storage/innobase/dict/dict0dict.cc
@@ -2139,93 +2139,6 @@ dict_col_name_is_reserved(
 	return(FALSE);
 }
 
-/****************************************************************//**
-Return maximum size of the node pointer record.
-@return maximum size of the record in bytes */
-ulint
-dict_index_node_ptr_max_size(
-/*=========================*/
-	const dict_index_t*	index)	/*!< in: index */
-{
-	ulint	comp;
-	ulint	i;
-	/* maximum possible storage size of a record */
-	ulint	rec_max_size;
-
-	if (dict_index_is_ibuf(index)) {
-		/* cannot estimate accurately */
-		/* This is universal index for change buffer.
-		The max size of the entry is about max key length * 2.
-		(index key + primary key to be inserted to the index)
-		(The max key length is UNIV_PAGE_SIZE / 16 * 3 at
-		 ha_innobase::max_supported_key_length(),
-		 considering MAX_KEY_LENGTH = 3072 at MySQL imposes
-		 the 3500 historical InnoDB value for 16K page size case.)
-		For the universal index, node_ptr contains most of the entry.
-		And 512 is enough to contain ibuf columns and meta-data */
-		return(UNIV_PAGE_SIZE / 8 * 3 + 512);
-	}
-
-	comp = dict_table_is_comp(index->table);
-
-	/* Each record has page_no, length of page_no and header. */
-	rec_max_size = comp
-		? REC_NODE_PTR_SIZE + 1 + REC_N_NEW_EXTRA_BYTES
-		: REC_NODE_PTR_SIZE + 2 + REC_N_OLD_EXTRA_BYTES;
-
-	if (comp) {
-		/* Include the "null" flags in the
-		maximum possible record size. */
-		rec_max_size += UT_BITS_IN_BYTES(index->n_nullable);
-	} else {
-		/* For each column, include a 2-byte offset and a
-		"null" flag. */
-		rec_max_size += 2 * index->n_fields;
-	}
-
-	/* Compute the maximum possible record size. */
-	for (i = 0; i < dict_index_get_n_unique_in_tree(index); i++) {
-		const dict_field_t*	field
-			= dict_index_get_nth_field(index, i);
-		const dict_col_t*	col
-			= dict_field_get_col(field);
-		ulint			field_max_size;
-		ulint			field_ext_max_size;
-
-		/* Determine the maximum length of the index field. */
-
-		field_max_size = dict_col_get_fixed_size(col, comp);
-		if (field_max_size) {
-			/* dict_index_add_col() should guarantee this */
-			ut_ad(!field->prefix_len
-			      || field->fixed_len == field->prefix_len);
-			/* Fixed lengths are not encoded
-			in ROW_FORMAT=COMPACT. */
-			rec_max_size += field_max_size;
-			continue;
-		}
-
-		field_max_size = dict_col_get_max_size(col);
-		field_ext_max_size = field_max_size < 256 ? 1 : 2;
-
-		if (field->prefix_len
-		    && field->prefix_len < field_max_size) {
-			field_max_size = field->prefix_len;
-		}
-
-		if (comp) {
-			/* Add the extra size for ROW_FORMAT=COMPACT.
-			For ROW_FORMAT=REDUNDANT, these bytes were
-			added to rec_max_size before this loop. */
-			rec_max_size += field_ext_max_size;
-		}
-
-		rec_max_size += field_max_size;
-	}
-
-	return(rec_max_size);
-}
-
 /****************************************************************//**
 If a record of this index might not fit on a single B-tree page,
 return TRUE.
diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h
index 9822b57d190..de997f0f684 100644
--- a/storage/innobase/include/dict0dict.h
+++ b/storage/innobase/include/dict0dict.h
@@ -1952,14 +1952,6 @@ const char*
 dict_tf_to_row_format_string(
 /*=========================*/
 	ulint	table_flag);		/*!< in: row format setting */
-/****************************************************************//**
-Return maximum size of the node pointer record.
-@return maximum size of the record in bytes */
-ulint
-dict_index_node_ptr_max_size(
-/*=========================*/
-	const dict_index_t*	index)	/*!< in: index */
-	MY_ATTRIBUTE((warn_unused_result));
 
 #define dict_col_is_virtual(col) (col)->is_virtual()
 

From de469a2f297b02c169d43ac1f6a0a6f63a913a81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Thu, 2 Aug 2018 17:35:24 +0300
Subject: [PATCH 099/164] MDEV-14637: Fix hang due to persistent statistics

Similar to the tables SYS_FOREIGN and SYS_FOREIGN_COLS,
the tables mysql.innodb_table_stats and mysql.innodb_index_stats
are updated by the InnoDB internal SQL parser, which fails to
enforce the size limits of the data. Due to this, it is possible
for InnoDB to hang when there are persistent statistics defined on
partitioned tables where the total length of table name,
partition name and subpartition name exceeds the incorrectly
defined limit VARCHAR(64). That column should have been defined
as VARCHAR(199).

btr_node_ptr_max_size(): Interpret the VARCHAR(64) as VARCHAR(199),
to prevent a hang in the case that the upgrade script has not been
run.

dict_table_schema_check(): Ignore difference in the length of the
table_name column.

ha_innobase::max_supported_key_length(): For innodb_page_size=4k,
return a larger value so that the table mysql.innodb_index_stats
can be created. This could allow "impossible" tables to be created,
such that it is not possible to insert anything into a secondary
index when both the secondary key and the primary key are long,
but this is the easiest and most consistent way. The Oracle fix
would only ignore the maximum length violation for the two
statistics tables.

os_file_get_status_posix(), os_file_get_status_win32(): Handle
ENAMETOOLONG as well.

This patch is based on the following change in MySQL 5.7.23.
Not all changes were applied, and our variant allows persistent
statistics to work without hangs even if the table definitions
were not upgraded.

From fdbdce701ab8145ae234c9d401109dff4e4106cb Mon Sep 17 00:00:00 2001
From: Aditya A <aditya.a@oracle.com>
Date: Thu, 17 May 2018 16:11:43 +0530
Subject: [PATCH] Bug #26390736 THE FIELD TABLE_NAME (VARCHAR(64)) FROM
           MYSQL.INNODB_TABLE_STATS CAN OVERFLOW.

    In mysql.innodb_index_stats and mysql.innodb_table_stats
    tables the table name column didn't take into consideration
    partition names which can be more than varchar(64).
---
 .../suite/funcs_1/r/is_columns_mysql.result    |  8 ++++----
 .../innodb_gis/r/create_spatial_index.result   |  2 +-
 .../innodb_gis/t/create_spatial_index.test     |  2 +-
 scripts/mysql_system_tables.sql                |  6 +++---
 scripts/mysql_system_tables_fix.sql            |  6 +++---
 storage/innobase/btr/btr0cur.cc                | 18 ++++++++++++++++++
 storage/innobase/dict/dict0dict.cc             | 13 +++++++++++--
 storage/innobase/dict/dict0stats.cc            |  8 +++-----
 storage/innobase/handler/ha_innodb.cc          | 11 ++++++-----
 storage/innobase/include/dict0stats.h          |  5 ++++-
 storage/innobase/os/os0file.cc                 |  6 ++++--
 11 files changed, 58 insertions(+), 27 deletions(-)

diff --git a/mysql-test/suite/funcs_1/r/is_columns_mysql.result b/mysql-test/suite/funcs_1/r/is_columns_mysql.result
index d021c73b339..25dbb185e44 100644
--- a/mysql-test/suite/funcs_1/r/is_columns_mysql.result
+++ b/mysql-test/suite/funcs_1/r/is_columns_mysql.result
@@ -124,13 +124,13 @@ def	mysql	innodb_index_stats	sample_size	7	NULL	YES	bigint	NULL	NULL	20	0	NULL	N
 def	mysql	innodb_index_stats	stat_description	8	NULL	NO	varchar	1024	3072	NULL	NULL	NULL	utf8	utf8_bin	varchar(1024)			select,insert,update,references		NEVER	NULL
 def	mysql	innodb_index_stats	stat_name	5	NULL	NO	varchar	64	192	NULL	NULL	NULL	utf8	utf8_bin	varchar(64)	PRI		select,insert,update,references		NEVER	NULL
 def	mysql	innodb_index_stats	stat_value	6	NULL	NO	bigint	NULL	NULL	20	0	NULL	NULL	NULL	bigint(20) unsigned			select,insert,update,references		NEVER	NULL
-def	mysql	innodb_index_stats	table_name	2	NULL	NO	varchar	64	192	NULL	NULL	NULL	utf8	utf8_bin	varchar(64)	PRI		select,insert,update,references		NEVER	NULL
+def	mysql	innodb_index_stats	table_name	2	NULL	NO	varchar	199	597	NULL	NULL	NULL	utf8	utf8_bin	varchar(199)	PRI		select,insert,update,references		NEVER	NULL
 def	mysql	innodb_table_stats	clustered_index_size	5	NULL	NO	bigint	NULL	NULL	20	0	NULL	NULL	NULL	bigint(20) unsigned			select,insert,update,references		NEVER	NULL
 def	mysql	innodb_table_stats	database_name	1	NULL	NO	varchar	64	192	NULL	NULL	NULL	utf8	utf8_bin	varchar(64)	PRI		select,insert,update,references		NEVER	NULL
 def	mysql	innodb_table_stats	last_update	3	current_timestamp()	NO	timestamp	NULL	NULL	NULL	NULL	0	NULL	NULL	timestamp		on update current_timestamp()	select,insert,update,references		NEVER	NULL
 def	mysql	innodb_table_stats	n_rows	4	NULL	NO	bigint	NULL	NULL	20	0	NULL	NULL	NULL	bigint(20) unsigned			select,insert,update,references		NEVER	NULL
 def	mysql	innodb_table_stats	sum_of_other_index_sizes	6	NULL	NO	bigint	NULL	NULL	20	0	NULL	NULL	NULL	bigint(20) unsigned			select,insert,update,references		NEVER	NULL
-def	mysql	innodb_table_stats	table_name	2	NULL	NO	varchar	64	192	NULL	NULL	NULL	utf8	utf8_bin	varchar(64)	PRI		select,insert,update,references		NEVER	NULL
+def	mysql	innodb_table_stats	table_name	2	NULL	NO	varchar	199	597	NULL	NULL	NULL	utf8	utf8_bin	varchar(199)	PRI		select,insert,update,references		NEVER	NULL
 def	mysql	plugin	dl	2	''	NO	varchar	128	384	NULL	NULL	NULL	utf8	utf8_general_ci	varchar(128)			select,insert,update,references		NEVER	NULL
 def	mysql	plugin	name	1	''	NO	varchar	64	192	NULL	NULL	NULL	utf8	utf8_general_ci	varchar(64)	PRI		select,insert,update,references		NEVER	NULL
 def	mysql	proc	body	11	NULL	NO	longblob	4294967295	4294967295	NULL	NULL	NULL	NULL	NULL	longblob			select,insert,update,references		NEVER	NULL
@@ -453,7 +453,7 @@ NULL	mysql	help_topic	help_category_id	smallint	NULL	NULL	NULL	NULL	smallint(5)
 NULL	mysql	index_stats	prefix_arity	int	NULL	NULL	NULL	NULL	int(11) unsigned
 NULL	mysql	index_stats	avg_frequency	decimal	NULL	NULL	NULL	NULL	decimal(12,4)
 3.0000	mysql	innodb_index_stats	database_name	varchar	64	192	utf8	utf8_bin	varchar(64)
-3.0000	mysql	innodb_index_stats	table_name	varchar	64	192	utf8	utf8_bin	varchar(64)
+3.0000	mysql	innodb_index_stats	table_name	varchar	199	597	utf8	utf8_bin	varchar(199)
 3.0000	mysql	innodb_index_stats	index_name	varchar	64	192	utf8	utf8_bin	varchar(64)
 NULL	mysql	innodb_index_stats	last_update	timestamp	NULL	NULL	NULL	NULL	timestamp
 3.0000	mysql	innodb_index_stats	stat_name	varchar	64	192	utf8	utf8_bin	varchar(64)
@@ -461,7 +461,7 @@ NULL	mysql	innodb_index_stats	stat_value	bigint	NULL	NULL	NULL	NULL	bigint(20) u
 NULL	mysql	innodb_index_stats	sample_size	bigint	NULL	NULL	NULL	NULL	bigint(20) unsigned
 3.0000	mysql	innodb_index_stats	stat_description	varchar	1024	3072	utf8	utf8_bin	varchar(1024)
 3.0000	mysql	innodb_table_stats	database_name	varchar	64	192	utf8	utf8_bin	varchar(64)
-3.0000	mysql	innodb_table_stats	table_name	varchar	64	192	utf8	utf8_bin	varchar(64)
+3.0000	mysql	innodb_table_stats	table_name	varchar	199	597	utf8	utf8_bin	varchar(199)
 NULL	mysql	innodb_table_stats	last_update	timestamp	NULL	NULL	NULL	NULL	timestamp
 NULL	mysql	innodb_table_stats	n_rows	bigint	NULL	NULL	NULL	NULL	bigint(20) unsigned
 NULL	mysql	innodb_table_stats	clustered_index_size	bigint	NULL	NULL	NULL	NULL	bigint(20) unsigned
diff --git a/mysql-test/suite/innodb_gis/r/create_spatial_index.result b/mysql-test/suite/innodb_gis/r/create_spatial_index.result
index 71d3092c3b7..80632affb67 100644
--- a/mysql-test/suite/innodb_gis/r/create_spatial_index.result
+++ b/mysql-test/suite/innodb_gis/r/create_spatial_index.result
@@ -1284,7 +1284,7 @@ CREATE SPATIAL INDEX idx2 ON t1(c1);
 DROP TABLE t1;
 CREATE TABLE mysql.innodb_table_stats (
 database_name varchar(64) COLLATE utf8_bin NOT NULL,
-table_name varchar(64) COLLATE utf8_bin NOT NULL,
+table_name varchar(199) COLLATE utf8_bin NOT NULL,
 last_update timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
 n_rows bigint(20) unsigned NOT NULL,
 clustered_index_size bigint(20) unsigned NOT NULL,
diff --git a/mysql-test/suite/innodb_gis/t/create_spatial_index.test b/mysql-test/suite/innodb_gis/t/create_spatial_index.test
index d25a2e79793..6a15b0f2d74 100644
--- a/mysql-test/suite/innodb_gis/t/create_spatial_index.test
+++ b/mysql-test/suite/innodb_gis/t/create_spatial_index.test
@@ -1166,7 +1166,7 @@ CREATE SPATIAL INDEX idx2 ON t1(c1);
 DROP TABLE t1;
 CREATE TABLE mysql.innodb_table_stats (
   database_name varchar(64) COLLATE utf8_bin NOT NULL,
-  table_name varchar(64) COLLATE utf8_bin NOT NULL,
+  table_name varchar(199) COLLATE utf8_bin NOT NULL,
   last_update timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
   n_rows bigint(20) unsigned NOT NULL,
   clustered_index_size bigint(20) unsigned NOT NULL,
diff --git a/scripts/mysql_system_tables.sql b/scripts/mysql_system_tables.sql
index 7b614163f46..c5792168213 100644
--- a/scripts/mysql_system_tables.sql
+++ b/scripts/mysql_system_tables.sql
@@ -1,4 +1,4 @@
--- Copyright (c) 2007, 2013, Oracle and/or its affiliates.
+-- Copyright (c) 2007, 2018, Oracle and/or its affiliates.
 -- Copyright (c) 2008, 2014, Monty Program Ab & SkySQL Ab
 -- 
 -- This program is free software; you can redistribute it and/or modify
@@ -105,7 +105,7 @@ CREATE TABLE IF NOT EXISTS event ( db char(64) CHARACTER SET utf8 COLLATE utf8_b
 
 SET @create_innodb_table_stats="CREATE TABLE IF NOT EXISTS innodb_table_stats (
 	database_name			VARCHAR(64) NOT NULL,
-	table_name			VARCHAR(64) NOT NULL,
+	table_name			VARCHAR(199) NOT NULL,
 	last_update			TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
 	n_rows				BIGINT UNSIGNED NOT NULL,
 	clustered_index_size		BIGINT UNSIGNED NOT NULL,
@@ -115,7 +115,7 @@ SET @create_innodb_table_stats="CREATE TABLE IF NOT EXISTS innodb_table_stats (
 
 SET @create_innodb_index_stats="CREATE TABLE IF NOT EXISTS innodb_index_stats (
 	database_name			VARCHAR(64) NOT NULL,
-	table_name			VARCHAR(64) NOT NULL,
+	table_name			VARCHAR(199) NOT NULL,
 	index_name			VARCHAR(64) NOT NULL,
 	last_update			TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
 	/* there are at least:
diff --git a/scripts/mysql_system_tables_fix.sql b/scripts/mysql_system_tables_fix.sql
index e724c3e2d23..db32834c4d6 100644
--- a/scripts/mysql_system_tables_fix.sql
+++ b/scripts/mysql_system_tables_fix.sql
@@ -697,13 +697,13 @@ set @str=replace(@str, "innodb_index_stats", "innodb_table_stats");
 prepare stmt from @str;
 execute stmt;
 
-# update timestamp fields in the innodb stat tables
-set @str="alter table mysql.innodb_index_stats modify last_update timestamp not null default current_timestamp on update current_timestamp";
+# update table_name and timestamp fields in the innodb stat tables
+set @str="alter table mysql.innodb_index_stats modify last_update timestamp not null default current_timestamp on update current_timestamp, modify table_name varchar(199)";
 set @str=if(@have_innodb <> 0, @str, "set @dummy = 0");
 prepare stmt from @str;
 execute stmt;
 
-set @str="alter table mysql.innodb_table_stats modify last_update timestamp not null default current_timestamp on update current_timestamp";
+set @str="alter table mysql.innodb_table_stats modify last_update timestamp not null default current_timestamp on update current_timestamp, modify table_name varchar(199)";
 set @str=if(@have_innodb <> 0, @str, "set @dummy = 0");
 prepare stmt from @str;
 execute stmt;
diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc
index cb03562b598..5af0a8b899a 100644
--- a/storage/innobase/btr/btr0cur.cc
+++ b/storage/innobase/btr/btr0cur.cc
@@ -65,6 +65,8 @@ Created 10/16/1994 Heikki Tuuri
 #include "lock0lock.h"
 #include "zlib.h"
 #include "srv0start.h"
+#include "mysql_com.h"
+#include "dict0stats.h"
 
 /** Buffered B-tree operation types, introduced as part of delete buffering. */
 enum btr_op_t {
@@ -771,6 +773,22 @@ static ulint btr_node_ptr_max_size(const dict_index_t* index)
 			rec_max_size += (srv_page_size == UNIV_PAGE_SIZE_MAX)
 				? REDUNDANT_REC_MAX_DATA_SIZE
 				: page_get_free_space_of_empty(FALSE) / 2;
+		} else if (field_max_size == NAME_LEN && i == 1
+			   && (!strcmp(index->table->name.m_name,
+				       TABLE_STATS_NAME)
+			       || !strcmp(index->table->name.m_name,
+					  INDEX_STATS_NAME))) {
+			ut_ad(!strcmp(field->name, "table_name"));
+			/* Interpret "table_name" as VARCHAR(199) even
+			if it was incorrectly defined as VARCHAR(64).
+			While the caller of ha_innobase enforces the
+			maximum length on any data written, the InnoDB
+			internal SQL parser will happily write as much
+			data as is provided. The purpose of this hack
+			is to avoid InnoDB hangs after persistent
+			statistics on partitioned tables are
+			deleted. */
+			field_max_size = 199 * SYSTEM_CHARSET_MBMAXLEN;
 		}
 		field_ext_max_size = field_max_size < 256 ? 1 : 2;
 
diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc
index a0d430e0257..fd7428001f8 100644
--- a/storage/innobase/dict/dict0dict.cc
+++ b/storage/innobase/dict/dict0dict.cc
@@ -6490,8 +6490,17 @@ dict_table_schema_check(
 		compare column types and flags */
 
 		/* check length for exact match */
-		if (req_schema->columns[i].len != table->cols[j].len) {
-
+		if (req_schema->columns[i].len == table->cols[j].len) {
+		} else if (!strcmp(req_schema->table_name, TABLE_STATS_NAME)
+			   || !strcmp(req_schema->table_name,
+				      INDEX_STATS_NAME)) {
+			ut_ad(table->cols[j].len < req_schema->columns[i].len);
+			ib::warn() << "Table " << req_schema->table_name
+				   << " has length mismatch in the"
+				   << " column name "
+				   << req_schema->columns[i].name
+				   << ".  Please run mysql_upgrade";
+		} else {
 			CREATE_TYPES_NAMES();
 
 			snprintf(errstr, errstr_sz,
diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc
index 98a846bcaba..2719c7e7222 100644
--- a/storage/innobase/dict/dict0stats.cc
+++ b/storage/innobase/dict/dict0stats.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2009, 2017, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2009, 2018, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2015, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -108,9 +108,7 @@ where n=1..n_uniq.
 @} */
 
 /* names of the tables from the persistent statistics storage */
-#define TABLE_STATS_NAME	"mysql/innodb_table_stats"
 #define TABLE_STATS_NAME_PRINT	"mysql.innodb_table_stats"
-#define INDEX_STATS_NAME	"mysql/innodb_index_stats"
 #define INDEX_STATS_NAME_PRINT	"mysql.innodb_index_stats"
 
 #ifdef UNIV_STATS_DEBUG
@@ -181,7 +179,7 @@ dict_stats_persistent_storage_check(
 			DATA_NOT_NULL, 192},
 
 		{"table_name", DATA_VARMYSQL,
-			DATA_NOT_NULL, 192},
+			DATA_NOT_NULL, 597},
 
 		{"last_update", DATA_FIXBINARY,
 			DATA_NOT_NULL, 4},
@@ -209,7 +207,7 @@ dict_stats_persistent_storage_check(
 			DATA_NOT_NULL, 192},
 
 		{"table_name", DATA_VARMYSQL,
-			DATA_NOT_NULL, 192},
+			DATA_NOT_NULL, 597},
 
 		{"index_name", DATA_VARMYSQL,
 			DATA_NOT_NULL, 192},
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index dccdeacb08f..aa6d9d48117 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -5459,15 +5459,16 @@ ha_innobase::max_supported_key_length() const
 
 	switch (UNIV_PAGE_SIZE) {
 	case 4096:
-		return(768);
+		/* Hack: allow mysql.innodb_index_stats to be created. */
+		/* FIXME: rewrite this API, and in sql_table.cc consider
+		that in index-organized tables (such as InnoDB), secondary
+		index records will be padded with the PRIMARY KEY, instead
+		of some short ROWID or record heap address. */
+		return(1173);
 	case 8192:
 		return(1536);
 	default:
-#ifdef WITH_WSREP
 		return(3500);
-#else
-		return(3500);
-#endif
 	}
 }
 
diff --git a/storage/innobase/include/dict0stats.h b/storage/innobase/include/dict0stats.h
index 03f0c0c6377..407e0221110 100644
--- a/storage/innobase/include/dict0stats.h
+++ b/storage/innobase/include/dict0stats.h
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2009, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2009, 2018, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -32,6 +32,9 @@ Created Jan 06, 2010 Vasil Dimov
 #include "dict0types.h"
 #include "trx0types.h"
 
+#define TABLE_STATS_NAME        "mysql/innodb_table_stats"
+#define INDEX_STATS_NAME        "mysql/innodb_index_stats"
+
 enum dict_stats_upd_option_t {
 	DICT_STATS_RECALC_PERSISTENT,/* (re) calculate the
 				statistics using a precise and slow
diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc
index ad8a76c5280..05131eb69b0 100644
--- a/storage/innobase/os/os0file.cc
+++ b/storage/innobase/os/os0file.cc
@@ -3307,7 +3307,8 @@ os_file_get_status_posix(
 {
 	int	ret = stat(path, statinfo);
 
-	if (ret && (errno == ENOENT || errno == ENOTDIR)) {
+	if (ret && (errno == ENOENT || errno == ENOTDIR
+		    || errno == ENAMETOOLONG)) {
 		/* file does not exist */
 
 		return(DB_NOT_FOUND);
@@ -4697,7 +4698,8 @@ os_file_get_status_win32(
 {
 	int	ret = _stat64(path, statinfo);
 
-	if (ret && (errno == ENOENT || errno == ENOTDIR)) {
+	if (ret && (errno == ENOENT || errno == ENOTDIR
+		    || errno == ENAMETOOLONG)) {
 		/* file does not exist */
 
 		return(DB_NOT_FOUND);

From 62ee2cd461596485dc8f349e3dfd87bf729fd500 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Thu, 2 Aug 2018 16:12:51 +0300
Subject: [PATCH 100/164] Fix a race between TRUNCATE and FOREIGN KEY check

This is a port of an Oracle fix.

No test case was provided by Oracle. It seems that to exploit this
bug, one would have to SET foreign_key_checks=0 before TRUNCATE,
and to concurrently run some DML statement that causes a foreign key
constraint to be checked.

commit 1f24c5aa2843fa548aa5c4b29c00f955e03e9f5b
Author: Aditya A <aditya.a@oracle.com>
Date:   Fri May 18 12:32:37 2018 +0530

    Bug #27208858 CONCURRENT DDL/DML ON FOREIGN KEYS CRASH IN
    PAGE_CUR_SEARCH_WITH_MATCH_BYTES
---
 storage/innobase/row/row0ins.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc
index 57796939be1..66948b83cda 100644
--- a/storage/innobase/row/row0ins.cc
+++ b/storage/innobase/row/row0ins.cc
@@ -1629,7 +1629,8 @@ row_ins_check_foreign_constraint(
 
 	if (check_table == NULL
 	    || !check_table->is_readable()
-	    || check_index == NULL) {
+	    || check_index == NULL
+	    || fil_space_get(check_table->space)->is_being_truncated) {
 
 		if (!srv_read_only_mode && check_ref) {
 			FILE*	ef = dict_foreign_err_file;

From b963cbaf4bcfe2c510c38edeacc2d6b1cae2d7a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 3 Aug 2018 11:49:49 +0300
Subject: [PATCH 101/164] Follow-up fix to MDEV-16865: InnoDB fts_query()
 ignores KILL

fts_query(): Remove a redundant condition (result will never be NULL),
and instead check if *result is NULL, to prevent SIGSEGV in
fts_query_free_result().
---
 storage/innobase/fts/fts0que.cc | 2 +-
 storage/xtradb/fts/fts0que.cc   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc
index 7d4cd7a3646..78521df75d9 100644
--- a/storage/innobase/fts/fts0que.cc
+++ b/storage/innobase/fts/fts0que.cc
@@ -4019,7 +4019,7 @@ fts_query(
 	if (trx_is_interrupted(trx)) {
 		error = DB_INTERRUPTED;
 		ut_free(lc_query_str);
-		if (result != NULL) {
+		if (*result) {
 			fts_query_free_result(*result);
 		}
 		goto func_exit;
diff --git a/storage/xtradb/fts/fts0que.cc b/storage/xtradb/fts/fts0que.cc
index 50f198401f9..9966656e772 100644
--- a/storage/xtradb/fts/fts0que.cc
+++ b/storage/xtradb/fts/fts0que.cc
@@ -4038,7 +4038,7 @@ fts_query(
 	if (trx_is_interrupted(trx)) {
 		error = DB_INTERRUPTED;
 		ut_free(lc_query_str);
-		if (result != NULL) {
+		if (*result) {
 			fts_query_free_result(*result);
 		}
 		goto func_exit;

From 9dfef6e29b98459a5077adb95d5b989d39e0db24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 3 Aug 2018 11:22:20 +0300
Subject: [PATCH 102/164] Fix -Wclass-memaccess warnings in InnoDB,XtraDB

---
 storage/innobase/buf/buf0buf.cc       | 2 +-
 storage/innobase/buf/buf0lru.cc       | 2 +-
 storage/innobase/include/dict0mem.h   | 3 +++
 storage/innobase/include/row0ftsort.h | 9 +++++++++
 storage/innobase/row/row0ftsort.cc    | 1 -
 storage/innobase/row/row0import.cc    | 4 ++--
 storage/innobase/trx/trx0trx.cc       | 2 +-
 storage/xtradb/buf/buf0buf.cc         | 2 +-
 storage/xtradb/buf/buf0lru.cc         | 2 +-
 storage/xtradb/include/dict0mem.h     | 3 +++
 storage/xtradb/include/row0ftsort.h   | 9 +++++++++
 storage/xtradb/row/row0ftsort.cc      | 1 -
 storage/xtradb/row/row0import.cc      | 4 ++--
 storage/xtradb/trx/trx0trx.cc         | 2 +-
 14 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc
index f534f8906f4..11dceacf592 100644
--- a/storage/innobase/buf/buf0buf.cc
+++ b/storage/innobase/buf/buf0buf.cc
@@ -1611,7 +1611,7 @@ buf_relocate(
 	}
 #endif /* UNIV_DEBUG */
 
-	memcpy(dpage, bpage, sizeof *dpage);
+	new (dpage) buf_page_t(*bpage);
 
 	ut_d(bpage->in_LRU_list = FALSE);
 	ut_d(bpage->in_page_hash = FALSE);
diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc
index 019dddb5863..5b0a2a7dd0c 100644
--- a/storage/innobase/buf/buf0lru.cc
+++ b/storage/innobase/buf/buf0lru.cc
@@ -1849,7 +1849,7 @@ func_exit:
 	} else if (buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE) {
 		b = buf_page_alloc_descriptor();
 		ut_a(b);
-		memcpy(b, bpage, sizeof *b);
+		new (b) buf_page_t(*bpage);
 	}
 
 	ut_ad(buf_pool_mutex_own(buf_pool));
diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h
index 394106235ef..872db7865d3 100644
--- a/storage/innobase/include/dict0mem.h
+++ b/storage/innobase/include/dict0mem.h
@@ -509,6 +509,9 @@ struct dict_field_t{
 	unsigned	fixed_len:10;	/*!< 0 or the fixed length of the
 					column if smaller than
 					DICT_ANTELOPE_MAX_INDEX_COL_LEN */
+
+	/** Zero-initialize all fields */
+	dict_field_t() : col(NULL), name(NULL), prefix_len(0), fixed_len(0) {}
 };
 
 /**********************************************************************//**
diff --git a/storage/innobase/include/row0ftsort.h b/storage/innobase/include/row0ftsort.h
index e949ba302b9..547937bac98 100644
--- a/storage/innobase/include/row0ftsort.h
+++ b/storage/innobase/include/row0ftsort.h
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2010, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2015, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -106,6 +107,14 @@ struct fts_tokenize_ctx {
 	ib_rbt_t*		cached_stopword;/*!< in: stopword list */
 	dfield_t		sort_field[FTS_NUM_FIELDS_SORT];
 						/*!< in: sort field */
+
+	fts_tokenize_ctx() :
+		processed_len(0), init_pos(0), buf_used(0),
+		rows_added(), cached_stopword(NULL), sort_field()
+	{
+		memset(rows_added, 0, sizeof rows_added);
+		memset(sort_field, 0, sizeof sort_field);
+	}
 };
 
 typedef struct fts_tokenize_ctx fts_tokenize_ctx_t;
diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc
index 8a3d4573879..757e268c3a7 100644
--- a/storage/innobase/row/row0ftsort.cc
+++ b/storage/innobase/row/row0ftsort.cc
@@ -638,7 +638,6 @@ fts_parallel_tokenization(
 	merge_file = psort_info->merge_file;
 	blob_heap = mem_heap_create(512);
 	memset(&doc, 0, sizeof(doc));
-	memset(&t_ctx, 0, sizeof(t_ctx));
 	memset(mycount, 0, FTS_NUM_AUX_INDEX * sizeof(int));
 
 	doc.charset = fts_index_get_charset(
diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc
index dad55205bc2..ea020fac56e 100644
--- a/storage/innobase/row/row0import.cc
+++ b/storage/innobase/row/row0import.cc
@@ -2620,8 +2620,6 @@ row_import_cfg_read_index_fields(
 
 	dict_field_t*	field = index->m_fields;
 
-	memset(field, 0x0, sizeof(*field) * n_fields);
-
 	for (ulint i = 0; i < n_fields; ++i, ++field) {
 		byte*		ptr = row;
 
@@ -2639,6 +2637,8 @@ row_import_cfg_read_index_fields(
 			return(DB_IO_ERROR);
 		}
 
+		new (field) dict_field_t();
+
 		field->prefix_len = mach_read_from_4(ptr);
 		ptr += sizeof(ib_uint32_t);
 
diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc
index 2401783648a..3d070e40570 100644
--- a/storage/innobase/trx/trx0trx.cc
+++ b/storage/innobase/trx/trx0trx.cc
@@ -140,7 +140,7 @@ trx_create(void)
 
 	trx->global_read_view_heap = mem_heap_create(256);
 
-	trx->xid.formatID = -1;
+	trx->xid.null();
 
 	trx->op_info = "";
 
diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc
index 4ad19c778ec..cc6774cd2b3 100644
--- a/storage/xtradb/buf/buf0buf.cc
+++ b/storage/xtradb/buf/buf0buf.cc
@@ -1718,7 +1718,7 @@ buf_relocate(
 	}
 #endif /* UNIV_DEBUG */
 
-	memcpy(dpage, bpage, sizeof *dpage);
+	new (dpage) buf_page_t(*bpage);
 
 	ut_d(bpage->in_LRU_list = FALSE);
 	ut_d(bpage->in_page_hash = FALSE);
diff --git a/storage/xtradb/buf/buf0lru.cc b/storage/xtradb/buf/buf0lru.cc
index caa008e8a96..e00bc3955de 100644
--- a/storage/xtradb/buf/buf0lru.cc
+++ b/storage/xtradb/buf/buf0lru.cc
@@ -2112,7 +2112,7 @@ not_freed:
 	}
 
 	if (b) {
-		memcpy(b, bpage, sizeof *b);
+		new (b) buf_page_t(*bpage);
 	}
 
 	if (!buf_LRU_block_remove_hashed(bpage, zip)) {
diff --git a/storage/xtradb/include/dict0mem.h b/storage/xtradb/include/dict0mem.h
index a4f810652e0..e6c0aa1f252 100644
--- a/storage/xtradb/include/dict0mem.h
+++ b/storage/xtradb/include/dict0mem.h
@@ -516,6 +516,9 @@ struct dict_field_t{
 	unsigned	fixed_len:10;	/*!< 0 or the fixed length of the
 					column if smaller than
 					DICT_ANTELOPE_MAX_INDEX_COL_LEN */
+
+	/** Zero-initialize all fields */
+	dict_field_t() : col(NULL), name(NULL), prefix_len(0), fixed_len(0) {}
 };
 
 /**********************************************************************//**
diff --git a/storage/xtradb/include/row0ftsort.h b/storage/xtradb/include/row0ftsort.h
index e949ba302b9..547937bac98 100644
--- a/storage/xtradb/include/row0ftsort.h
+++ b/storage/xtradb/include/row0ftsort.h
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2010, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2015, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -106,6 +107,14 @@ struct fts_tokenize_ctx {
 	ib_rbt_t*		cached_stopword;/*!< in: stopword list */
 	dfield_t		sort_field[FTS_NUM_FIELDS_SORT];
 						/*!< in: sort field */
+
+	fts_tokenize_ctx() :
+		processed_len(0), init_pos(0), buf_used(0),
+		rows_added(), cached_stopword(NULL), sort_field()
+	{
+		memset(rows_added, 0, sizeof rows_added);
+		memset(sort_field, 0, sizeof sort_field);
+	}
 };
 
 typedef struct fts_tokenize_ctx fts_tokenize_ctx_t;
diff --git a/storage/xtradb/row/row0ftsort.cc b/storage/xtradb/row/row0ftsort.cc
index 2292083689c..bb9821d4484 100644
--- a/storage/xtradb/row/row0ftsort.cc
+++ b/storage/xtradb/row/row0ftsort.cc
@@ -641,7 +641,6 @@ fts_parallel_tokenization(
 	merge_file = psort_info->merge_file;
 	blob_heap = mem_heap_create(512);
 	memset(&doc, 0, sizeof(doc));
-	memset(&t_ctx, 0, sizeof(t_ctx));
 	memset(mycount, 0, FTS_NUM_AUX_INDEX * sizeof(int));
 
 	doc.charset = fts_index_get_charset(
diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc
index dad55205bc2..ea020fac56e 100644
--- a/storage/xtradb/row/row0import.cc
+++ b/storage/xtradb/row/row0import.cc
@@ -2620,8 +2620,6 @@ row_import_cfg_read_index_fields(
 
 	dict_field_t*	field = index->m_fields;
 
-	memset(field, 0x0, sizeof(*field) * n_fields);
-
 	for (ulint i = 0; i < n_fields; ++i, ++field) {
 		byte*		ptr = row;
 
@@ -2639,6 +2637,8 @@ row_import_cfg_read_index_fields(
 			return(DB_IO_ERROR);
 		}
 
+		new (field) dict_field_t();
+
 		field->prefix_len = mach_read_from_4(ptr);
 		ptr += sizeof(ib_uint32_t);
 
diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc
index 8e62a7dba46..5f5ecba837e 100644
--- a/storage/xtradb/trx/trx0trx.cc
+++ b/storage/xtradb/trx/trx0trx.cc
@@ -276,7 +276,7 @@ trx_create(void)
 	trx->distinct_page_access_hash = NULL;
 	trx->take_stats = FALSE;
 
-	trx->xid.formatID = -1;
+	trx->xid.null();
 
 	trx->op_info = "";
 

From 769f6d2db72dd70ff2fef69bed02fb4124a682ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 3 Aug 2018 12:21:13 +0300
Subject: [PATCH 103/164] Fix -Wclass-memaccess in WSREP,InnoDB,XtraDB

---
 extra/mariabackup/backup_copy.cc       | 19 ++++++++++++++++---
 extra/mariabackup/wsrep.cc             |  3 +--
 sql/handler.h                          |  4 ++--
 sql/wsrep_xid.cc                       |  5 ++---
 storage/innobase/dict/dict0stats_bg.cc | 11 +++--------
 storage/innobase/trx/trx0sys.cc        | 12 +++++++-----
 storage/innobase/trx/trx0trx.cc        |  3 +--
 storage/xtradb/dict/dict0stats_bg.cc   | 14 ++++----------
 storage/xtradb/trx/trx0sys.cc          | 12 +++++++-----
 storage/xtradb/trx/trx0trx.cc          |  3 +--
 10 files changed, 44 insertions(+), 42 deletions(-)

diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc
index 7cdc6b8e3a6..895ef744ea1 100644
--- a/extra/mariabackup/backup_copy.cc
+++ b/extra/mariabackup/backup_copy.cc
@@ -460,6 +460,21 @@ struct datafile_cur_t {
 	size_t		buf_size;
 	size_t		buf_read;
 	size_t		buf_offset;
+
+	explicit datafile_cur_t(const char* filename = NULL) :
+		file(), thread_n(0), orig_buf(NULL), buf(NULL), buf_size(0),
+		buf_read(0), buf_offset(0)
+	{
+		memset(rel_path, 0, sizeof rel_path);
+		if (filename) {
+			strncpy(abs_path, filename, sizeof abs_path);
+			abs_path[(sizeof abs_path) - 1] = 0;
+		} else {
+			abs_path[0] = '\0';
+		}
+		rel_path[0] = '\0';
+		memset(&statinfo, 0, sizeof statinfo);
+	}
 };
 
 static
@@ -478,9 +493,7 @@ datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n)
 {
 	ulint		success;
 
-	memset(cursor, 0, sizeof(datafile_cur_t));
-
-	strncpy(cursor->abs_path, file, sizeof(cursor->abs_path));
+	new (cursor) datafile_cur_t(file);
 
 	/* Get the relative path for the destination tablespace name, i.e. the
 	one that can be appended to the backup root directory. Non-system
diff --git a/extra/mariabackup/wsrep.cc b/extra/mariabackup/wsrep.cc
index 3baa9e660d7..7b0b26d803a 100644
--- a/extra/mariabackup/wsrep.cc
+++ b/extra/mariabackup/wsrep.cc
@@ -179,8 +179,7 @@ xb_write_galera_info(bool incremental_prepare)
 		return;
 	}
 
-	memset(&xid, 0, sizeof(xid));
-	xid.formatID = -1;
+	xid.null();
 
 	if (!trx_sys_read_wsrep_checkpoint(&xid)) {
 
diff --git a/sql/handler.h b/sql/handler.h
index d17bccab25c..42429e9dcbe 100644
--- a/sql/handler.h
+++ b/sql/handler.h
@@ -2,7 +2,7 @@
 #define HANDLER_INCLUDED
 /*
    Copyright (c) 2000, 2016, Oracle and/or its affiliates.
-   Copyright (c) 2009, 2016, MariaDB
+   Copyright (c) 2009, 2018, MariaDB
 
    This program is free software; you can redistribute it and/or
    modify it under the terms of the GNU General Public License
@@ -602,7 +602,7 @@ struct xid_t {
     bqual_length= b;
     memcpy(data, d, g+b);
   }
-  bool is_null() { return formatID == -1; }
+  bool is_null() const { return formatID == -1; }
   void null() { formatID= -1; }
   my_xid quick_get_my_xid()
   {
diff --git a/sql/wsrep_xid.cc b/sql/wsrep_xid.cc
index f7ea836d494..132956e88b3 100644
--- a/sql/wsrep_xid.cc
+++ b/sql/wsrep_xid.cc
@@ -129,12 +129,11 @@ void wsrep_get_SE_checkpoint(wsrep_uuid_t& uuid, wsrep_seqno_t& seqno)
   seqno= WSREP_SEQNO_UNDEFINED;
 
   XID xid;
-  memset(&xid, 0, sizeof(xid));
-  xid.formatID= -1;
+  xid.null();
 
   wsrep_get_SE_checkpoint(xid);
 
-  if (xid.formatID == -1) return; // nil XID
+  if (xid.is_null()) return;
 
   if (!wsrep_is_wsrep_xid(&xid))
   {
diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc
index 6f2f96c452e..9394ca2830d 100644
--- a/storage/innobase/dict/dict0stats_bg.cc
+++ b/storage/innobase/dict/dict0stats_bg.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2012, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation.
+Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -95,10 +95,7 @@ dict_stats_pool_init()
 /*****************************************************************//**
 Free the resources occupied by the recalc pool, called once during
 thread de-initialization. */
-static
-void
-dict_stats_pool_deinit()
-/*====================*/
+static void dict_stats_pool_deinit()
 {
 	ut_ad(!srv_read_only_mode);
 
@@ -114,9 +111,7 @@ dict_stats_pool_deinit()
         */
 	recalc_pool_t recalc_empty_pool;
 	defrag_pool_t defrag_empty_pool;
-	memset(&recalc_empty_pool, 0, sizeof(recalc_pool_t));
-	memset(&defrag_empty_pool, 0, sizeof(defrag_pool_t));
-        recalc_pool.swap(recalc_empty_pool);
+	recalc_pool.swap(recalc_empty_pool);
 	defrag_pool.swap(defrag_empty_pool);
 }
 
diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc
index 0246eaf32a8..da2e4917cb0 100644
--- a/storage/innobase/trx/trx0sys.cc
+++ b/storage/innobase/trx/trx0sys.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation.
+Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -347,7 +347,7 @@ trx_sys_update_wsrep_checkpoint(
 	mtr_t*		mtr)
 {
 #ifdef UNIV_DEBUG
-	if (xid->formatID != -1
+	if (!xid->is_null()
 	    && mach_read_from_4(sys_header + TRX_SYS_WSREP_XID_INFO
 			+ TRX_SYS_WSREP_XID_MAGIC_N_FLD)
 		== TRX_SYS_WSREP_XID_MAGIC_N) {
@@ -368,7 +368,7 @@ trx_sys_update_wsrep_checkpoint(
 #endif /* UNIV_DEBUG */
 
 	ut_ad(xid && mtr);
-	ut_a(xid->formatID == -1 || wsrep_is_wsrep_xid((const XID *)xid));
+	ut_a(xid->is_null() || wsrep_is_wsrep_xid((const XID *)xid));
 
 	if (mach_read_from_4(sys_header + TRX_SYS_WSREP_XID_INFO
 			+ TRX_SYS_WSREP_XID_MAGIC_N_FLD)
@@ -417,8 +417,10 @@ trx_sys_read_wsrep_checkpoint(XID* xid)
 	if ((magic = mach_read_from_4(sys_header + TRX_SYS_WSREP_XID_INFO
 				+ TRX_SYS_WSREP_XID_MAGIC_N_FLD))
 		!= TRX_SYS_WSREP_XID_MAGIC_N) {
-		memset(xid, 0, sizeof(*xid));
-		xid->formatID = -1;
+		xid->null();
+		xid->gtrid_length = 0;
+		xid->bqual_length = 0;
+		memset(xid->data, 0, sizeof xid->data);
 		trx_sys_update_wsrep_checkpoint(xid, sys_header, &mtr);
 		mtr_commit(&mtr);
 		return false;
diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc
index 3c86adb84b4..58dc9445546 100644
--- a/storage/innobase/trx/trx0trx.cc
+++ b/storage/innobase/trx/trx0trx.cc
@@ -851,8 +851,7 @@ trx_start_low(
 	}
 
 #ifdef WITH_WSREP
-        memset(&trx->xid, 0, sizeof(trx->xid));
-        trx->xid.formatID = -1;
+        trx->xid.null();
 #endif /* WITH_WSREP */
 
 	/* The initial value for trx->no: TRX_ID_MAX is used in
diff --git a/storage/xtradb/dict/dict0stats_bg.cc b/storage/xtradb/dict/dict0stats_bg.cc
index 884f62103f5..9394ca2830d 100644
--- a/storage/xtradb/dict/dict0stats_bg.cc
+++ b/storage/xtradb/dict/dict0stats_bg.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2012, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation.
+Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -95,17 +95,13 @@ dict_stats_pool_init()
 /*****************************************************************//**
 Free the resources occupied by the recalc pool, called once during
 thread de-initialization. */
-static
-void
-dict_stats_pool_deinit()
-/*===========================*/
+static void dict_stats_pool_deinit()
 {
 	ut_ad(!srv_read_only_mode);
 
 	recalc_pool.clear();
 	defrag_pool.clear();
-
- /*
+        /*
           recalc_pool may still have its buffer allocated. It will free it when
           its destructor is called.
           The problem is, memory leak detector is run before the recalc_pool's
@@ -115,9 +111,7 @@ dict_stats_pool_deinit()
         */
 	recalc_pool_t recalc_empty_pool;
 	defrag_pool_t defrag_empty_pool;
-	memset(&recalc_empty_pool, 0, sizeof(recalc_pool_t));
-	memset(&defrag_empty_pool, 0, sizeof(defrag_pool_t));
-        recalc_pool.swap(recalc_empty_pool);
+	recalc_pool.swap(recalc_empty_pool);
 	defrag_pool.swap(defrag_empty_pool);
 }
 
diff --git a/storage/xtradb/trx/trx0sys.cc b/storage/xtradb/trx/trx0sys.cc
index 6108ab7ab94..30660df01f6 100644
--- a/storage/xtradb/trx/trx0sys.cc
+++ b/storage/xtradb/trx/trx0sys.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation.
+Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -351,7 +351,7 @@ trx_sys_update_wsrep_checkpoint(
 	mtr_t*		mtr)
 {
 #ifdef UNIV_DEBUG
-	if (xid->formatID != -1
+	if (!xid->is_null()
 	    && mach_read_from_4(sys_header + TRX_SYS_WSREP_XID_INFO
 			+ TRX_SYS_WSREP_XID_MAGIC_N_FLD)
 		== TRX_SYS_WSREP_XID_MAGIC_N) {
@@ -372,7 +372,7 @@ trx_sys_update_wsrep_checkpoint(
 #endif /* UNIV_DEBUG */
 
 	ut_ad(xid && mtr);
-	ut_a(xid->formatID == -1 || wsrep_is_wsrep_xid((const XID *)xid));
+	ut_a(xid->is_null() || wsrep_is_wsrep_xid((const XID *)xid));
 
 	if (mach_read_from_4(sys_header + TRX_SYS_WSREP_XID_INFO
 			+ TRX_SYS_WSREP_XID_MAGIC_N_FLD)
@@ -421,8 +421,10 @@ trx_sys_read_wsrep_checkpoint(XID* xid)
 	if ((magic = mach_read_from_4(sys_header + TRX_SYS_WSREP_XID_INFO
 				+ TRX_SYS_WSREP_XID_MAGIC_N_FLD))
 		!= TRX_SYS_WSREP_XID_MAGIC_N) {
-		memset(xid, 0, sizeof(*xid));
-		xid->formatID = -1;
+		xid->null();
+		xid->gtrid_length = 0;
+		xid->bqual_length = 0;
+		memset(xid->data, 0, sizeof xid->data);
 		trx_sys_update_wsrep_checkpoint(xid, sys_header, &mtr);
 		mtr_commit(&mtr);
 		return false;
diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc
index b51afa4febd..6a033fbeb09 100644
--- a/storage/xtradb/trx/trx0trx.cc
+++ b/storage/xtradb/trx/trx0trx.cc
@@ -1041,8 +1041,7 @@ trx_start_low(
 	}
 
 #ifdef WITH_WSREP
-        memset(&trx->xid, 0, sizeof(trx->xid));
-        trx->xid.formatID = -1;
+        trx->xid.null();
 #endif /* WITH_WSREP */
 
 	/* The initial value for trx->no: TRX_ID_MAX is used in

From 9d42eb5e28303adb4d9467ac3e9c3be35d46e643 Mon Sep 17 00:00:00 2001
From: Oleksandr Byelkin <sanja@mariadb.com>
Date: Fri, 3 Aug 2018 12:30:50 +0300
Subject: [PATCH 104/164] Disable an unstable test

---
 mysql-test/suite/galera/disabled.def | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def
index 397ad22dab8..2ec3787c3b6 100644
--- a/mysql-test/suite/galera/disabled.def
+++ b/mysql-test/suite/galera/disabled.def
@@ -30,3 +30,4 @@ query_cache : MDEV-15805 Test failure on galera.query_cache
 MW-416 : MDEV-13549 Galera test failures
 galera_wan : MDEV-13549 Galera test failures
 MW-388 : MDEV-13549 Galera test failures
+galera_sst_mysqldump_with_key : MDEV-16890 Galera test failure

From 391e60b2dbdf98066cc8c8c2ee0a58e7de879d4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 3 Aug 2018 13:06:03 +0300
Subject: [PATCH 105/164] Fix -Wclass-memaccess warnings in InnoDB

---
 storage/innobase/buf/buf0buf.cc     |  4 ++--
 storage/innobase/gis/gis0rtree.cc   |  4 ++--
 storage/innobase/gis/gis0sea.cc     |  2 +-
 storage/innobase/include/btr0cur.h  | 20 ++++++++++++++++++++
 storage/innobase/include/btr0pcur.h | 12 ++++++++++++
 storage/innobase/row/row0uins.cc    |  4 +---
 6 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc
index 6ff71a150a2..724fea07ba2 100644
--- a/storage/innobase/buf/buf0buf.cc
+++ b/storage/innobase/buf/buf0buf.cc
@@ -2214,8 +2214,8 @@ buf_page_realloc(
 	if (buf_page_can_relocate(&block->page)) {
 		mutex_enter(&new_block->mutex);
 
-		memcpy(new_block->frame, block->frame, UNIV_PAGE_SIZE);
-		memcpy(&new_block->page, &block->page, sizeof block->page);
+		memcpy(new_block->frame, block->frame, srv_page_size);
+		new (&new_block->page) buf_page_t(block->page);
 
 		/* relocate LRU list */
 		ut_ad(block->page.in_LRU_list);
diff --git a/storage/innobase/gis/gis0rtree.cc b/storage/innobase/gis/gis0rtree.cc
index d45e40c8151..a3790f5440e 100644
--- a/storage/innobase/gis/gis0rtree.cc
+++ b/storage/innobase/gis/gis0rtree.cc
@@ -662,7 +662,7 @@ rtr_adjust_upper_level(
 
 	/* Create a memory heap where the data tuple is stored */
 	heap = mem_heap_create(1024);
-	memset(&cursor, 0, sizeof(cursor));
+	cursor.init();
 
 	cursor.thr = sea_cur->thr;
 
@@ -1379,7 +1379,7 @@ rtr_ins_enlarge_mbr(
 		rtr_page_cal_mbr(index, block, &new_mbr, heap);
 
 		/* Get father block. */
-		memset(&cursor, 0, sizeof(cursor));
+		cursor.init();
 		offsets = rtr_page_get_father_block(
 			NULL, heap, index, block, mtr, btr_cur, &cursor);
 
diff --git a/storage/innobase/gis/gis0sea.cc b/storage/innobase/gis/gis0sea.cc
index c45f2ecec59..9167cf9511b 100644
--- a/storage/innobase/gis/gis0sea.cc
+++ b/storage/innobase/gis/gis0sea.cc
@@ -1548,7 +1548,7 @@ rtr_copy_buf(
 	will be copied. It is also undefined what will happen with the
 	newly memcpy()ed mutex if the source mutex was acquired by
 	(another) thread while it was copied. */
-	memcpy(&matches->block.page, &block->page, sizeof(buf_page_t));
+	new (&matches->block.page) buf_page_t(block->page);
 	matches->block.frame = block->frame;
 	matches->block.unzip_LRU = block->unzip_LRU;
 
diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h
index d346a047eac..1df382bb995 100644
--- a/storage/innobase/include/btr0cur.h
+++ b/storage/innobase/include/btr0cur.h
@@ -926,6 +926,26 @@ struct btr_cur_t {
 	rtr_info_t*	rtr_info;	/*!< rtree search info */
 	btr_cur_t():thr(NULL), rtr_info(NULL) {}
 					/* default values */
+	/** Zero-initialize all fields */
+	void init()
+	{
+		index = NULL;
+		memset(&page_cur, 0, sizeof page_cur);
+		purge_node = NULL;
+		left_block = NULL;
+		thr = NULL;
+		flag = btr_cur_method(0);
+		tree_height = 0;
+		up_match = 0;
+		up_bytes = 0;
+		low_match = 0;
+		low_bytes = 0;
+		n_fields = 0;
+		n_bytes = 0;
+		fold = 0;
+		path_arr = NULL;
+		rtr_info = NULL;
+	}
 };
 
 /******************************************************//**
diff --git a/storage/innobase/include/btr0pcur.h b/storage/innobase/include/btr0pcur.h
index f9f132b1733..b84d9840a28 100644
--- a/storage/innobase/include/btr0pcur.h
+++ b/storage/innobase/include/btr0pcur.h
@@ -543,6 +543,18 @@ struct btr_pcur_t{
 	/** old_rec_buf size if old_rec_buf is not NULL */
 	ulint		buf_size;
 
+	btr_pcur_t() :
+		btr_cur(), latch_mode(0), old_stored(false), old_rec(NULL),
+		old_n_fields(0), rel_pos(btr_pcur_pos_t(0)),
+		block_when_stored(NULL),
+		modify_clock(0), withdraw_clock(0),
+		pos_state(BTR_PCUR_NOT_POSITIONED),
+		search_mode(PAGE_CUR_UNSUPP), trx_if_known(NULL),
+		old_rec_buf(NULL), buf_size(0)
+	{
+		btr_cur.init();
+	}
+
 	/** Return the index of this persistent cursor */
 	dict_index_t*	index() const { return(btr_cur.index); }
 };
diff --git a/storage/innobase/row/row0uins.cc b/storage/innobase/row/row0uins.cc
index b3e53cab028..d50d86ac338 100644
--- a/storage/innobase/row/row0uins.cc
+++ b/storage/innobase/row/row0uins.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation.
+Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -202,8 +202,6 @@ row_undo_ins_remove_sec_low(
 	enum row_search_result	search_result;
 	const bool		modify_leaf = mode == BTR_MODIFY_LEAF;
 
-	memset(&pcur, 0, sizeof(pcur));
-
 	row_mtr_start(&mtr, index, !modify_leaf);
 
 	if (modify_leaf) {

From 7b145fae13d179f5cdc2798e7f892bbae79093ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 3 Aug 2018 11:24:05 +0300
Subject: [PATCH 106/164] mariabackup: Use snprintf() instead of sprintf()

---
 extra/mariabackup/xtrabackup.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc
index 390cdab52f2..9c961af2583 100644
--- a/extra/mariabackup/xtrabackup.cc
+++ b/extra/mariabackup/xtrabackup.cc
@@ -1370,7 +1370,7 @@ static int prepare_export()
   // which is* unfortunately* still necessary to get mysqld up
   if (strncmp(orig_argv1,"--defaults-file=",16) == 0)
   {
-    sprintf(cmdline, 
+    snprintf(cmdline, sizeof cmdline,
      IF_WIN("\"","") "\"%s\" --mysqld \"%s\" "
       " --defaults-extra-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
       " --innodb --innodb-fast-shutdown=0"

From be370efac43c26168e13034fa9b0bb0116ca95b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 3 Aug 2018 14:36:38 +0300
Subject: [PATCH 107/164] Do not declare an unused variable

---
 storage/innobase/lock/lock0lock.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc
index 1ce798fcd43..61f62d1e054 100644
--- a/storage/innobase/lock/lock0lock.cc
+++ b/storage/innobase/lock/lock0lock.cc
@@ -1805,7 +1805,10 @@ lock_rec_enqueue_waiting(
 		lock_prdt_set_prdt(lock, prdt);
 	}
 
-	if (const trx_t* victim =
+	if (
+#ifdef UNIV_DEBUG
+	    const trx_t* victim =
+#endif
 	    DeadlockChecker::check_and_resolve(lock, trx)) {
 		ut_ad(victim == trx);
 		lock_reset_lock_and_trx_wait(lock);

From 701f0b8e366f957e8256e4741ca48424c84b7234 Mon Sep 17 00:00:00 2001
From: Oleksandr Byelkin <sanja@mariadb.com>
Date: Fri, 3 Aug 2018 14:37:55 +0200
Subject: [PATCH 108/164] Fix gcc 7.3 compiler warnings.

---
 sql/opt_range.cc     |  5 +++--
 sql/opt_subselect.cc |  6 +++---
 sql/sql_acl.cc       |  4 +++-
 sql/sql_class.h      |  4 ++++
 sql/sql_explain.cc   |  2 +-
 sql/sql_partition.cc | 12 ++++++++++++
 sql/table.cc         |  2 +-
 7 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/sql/opt_range.cc b/sql/opt_range.cc
index 0315b91b8be..1e29efda0d1 100644
--- a/sql/opt_range.cc
+++ b/sql/opt_range.cc
@@ -7307,7 +7307,8 @@ SEL_TREE *Item_bool_func::get_full_func_mm_tree(RANGE_OPT_PARAM *param,
 		          param->current_table);
 #ifdef HAVE_SPATIAL
   Field::geometry_type sav_geom_type;
-  if (field_item->field->type() == MYSQL_TYPE_GEOMETRY)
+  const bool geometry= field_item->field->type() == MYSQL_TYPE_GEOMETRY;
+  if (geometry)
   {
     sav_geom_type= ((Field_geom*) field_item->field)->geom_type;
     /* We have to be able to store all sorts of spatial features here */
@@ -7342,7 +7343,7 @@ SEL_TREE *Item_bool_func::get_full_func_mm_tree(RANGE_OPT_PARAM *param,
   }
 
 #ifdef HAVE_SPATIAL
-  if (field_item->field->type() == MYSQL_TYPE_GEOMETRY)
+  if (geometry)
   {
     ((Field_geom*) field_item->field)->geom_type= sav_geom_type;
   }
diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc
index ca5398197bd..85f5645949e 100644
--- a/sql/opt_subselect.cc
+++ b/sql/opt_subselect.cc
@@ -2776,8 +2776,8 @@ void advance_sj_state(JOIN *join, table_map remaining_tables, uint idx,
     LooseScan detector in best_access_path)
   */
   remaining_tables &= ~new_join_tab->table->map;
-  table_map dups_producing_tables, prev_dups_producing_tables,
-            prev_sjm_lookup_tables;
+  table_map dups_producing_tables, prev_dups_producing_tables= 0,
+            prev_sjm_lookup_tables= 0;
 
   if (idx == join->const_tables)
     dups_producing_tables= 0;
@@ -2788,7 +2788,7 @@ void advance_sj_state(JOIN *join, table_map remaining_tables, uint idx,
   if ((emb_sj_nest= new_join_tab->emb_sj_nest))
     dups_producing_tables |= emb_sj_nest->sj_inner_tables;
 
-  Semi_join_strategy_picker **strategy, **prev_strategy;
+  Semi_join_strategy_picker **strategy, **prev_strategy= NULL;
   if (idx == join->const_tables)
   {
     /* First table, initialize pickers */
diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc
index b3269660a7c..3fab00a46df 100644
--- a/sql/sql_acl.cc
+++ b/sql/sql_acl.cc
@@ -2891,7 +2891,8 @@ int acl_set_default_role(THD *thd, const char *host, const char *user,
   ulong query_length= 0;
   bool clear_role= FALSE;
   char buff[512];
-  enum_binlog_format save_binlog_format;
+  enum_binlog_format save_binlog_format=
+    thd->get_current_stmt_binlog_format();
   const CSET_STRING query_save __attribute__((unused)) = thd->query_string;
 
   DBUG_ENTER("acl_set_default_role");
@@ -2922,6 +2923,7 @@ int acl_set_default_role(THD *thd, const char *host, const char *user,
   if (WSREP(thd) && !IF_WSREP(thd->wsrep_applier, 0))
   {
     thd->set_query(buff, query_length, system_charset_info);
+    // Attention!!! here is implicit goto error;
     WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, (char*)"user", NULL);
   }
 
diff --git a/sql/sql_class.h b/sql/sql_class.h
index ab721fd0585..53e451511d5 100644
--- a/sql/sql_class.h
+++ b/sql/sql_class.h
@@ -3562,6 +3562,10 @@ public:
     *format= (enum_binlog_format) variables.binlog_format;
     *current_format= current_stmt_binlog_format;
   }
+  inline enum_binlog_format get_current_stmt_binlog_format()
+  {
+    return current_stmt_binlog_format;
+  }
   inline void set_binlog_format(enum_binlog_format format,
                                 enum_binlog_format current_format)
   {
diff --git a/sql/sql_explain.cc b/sql/sql_explain.cc
index 82107f9b922..a1d6764d8e4 100644
--- a/sql/sql_explain.cc
+++ b/sql/sql_explain.cc
@@ -895,7 +895,7 @@ void Explain_select::print_explain_json(Explain_query *query,
       {
         Sort_and_group_tracker::Iterator iter(&ops_tracker);
         enum_qep_action action;
-        Filesort_tracker *fs_tracker;
+        Filesort_tracker *fs_tracker= NULL;
 
         while ((action= iter.get_next(&fs_tracker)) != EXPL_ACTION_EOF)
         {
diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc
index 65c76127de9..05ef69e5795 100644
--- a/sql/sql_partition.cc
+++ b/sql/sql_partition.cc
@@ -8313,6 +8313,12 @@ int create_partition_name(char *out, size_t outlen, const char *in1,
     end= strxnmov(out, outlen-1, in1, "#P#", transl_part, "#TMP#", NullS);
   else if (name_variant == RENAMED_PART_NAME)
     end= strxnmov(out, outlen-1, in1, "#P#", transl_part, "#REN#", NullS);
+  else
+  {
+    DBUG_ASSERT(0);
+    out[0]= 0;
+    end= out + (outlen-1);
+  }
   if (end - out == static_cast<ptrdiff_t>(outlen-1))
   {
     my_error(ER_PATH_LENGTH, MYF(0), longest_str(in1, transl_part));
@@ -8355,6 +8361,12 @@ int create_subpartition_name(char *out, size_t outlen,
   else if (name_variant == RENAMED_PART_NAME)
     end= strxnmov(out, outlen-1, in1, "#P#", transl_part_name,
                   "#SP#", transl_subpart_name, "#REN#", NullS);
+  else
+  {
+    DBUG_ASSERT(0);
+    out[0]= 0;
+    end= out + (outlen-1);
+  }
   if (end - out == static_cast<ptrdiff_t>(outlen-1))
   {
     my_error(ER_PATH_LENGTH, MYF(0),
diff --git a/sql/table.cc b/sql/table.cc
index b267df96139..b3a1c2eaa3d 100644
--- a/sql/table.cc
+++ b/sql/table.cc
@@ -1723,7 +1723,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
     keyinfo= share->key_info;
     uint primary_key= my_strcasecmp(system_charset_info, share->keynames.type_names[0],
                                     primary_key_name) ? MAX_KEY : 0;
-    KEY* key_first_info;
+    KEY* key_first_info= NULL;
 
     if (primary_key >= MAX_KEY && keyinfo->flags & HA_NOSAME)
     {

From e6a808bec790fdbbf1dc0a6b03fb6c8b6e41dc01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 3 Aug 2018 15:40:13 +0300
Subject: [PATCH 109/164] Fix heap-use-after-free in debug code

rw_lock_get_debug_info(): Remove. This function is inherently unsafe
to use, because the copied pointers can become stale between
rw_lock_debug_mutex_exit() and the dereferencing of the pointer in
the caller.
---
 storage/innobase/sync/sync0rw.cc | 99 +++++++++++---------------------
 1 file changed, 34 insertions(+), 65 deletions(-)

diff --git a/storage/innobase/sync/sync0rw.cc b/storage/innobase/sync/sync0rw.cc
index c985cf9ee82..0ad646f06b0 100644
--- a/storage/innobase/sync/sync0rw.cc
+++ b/storage/innobase/sync/sync0rw.cc
@@ -1047,86 +1047,52 @@ rw_lock_own(
 	return(FALSE);
 }
 
-/** For collecting the debug information for a thread's rw-lock */
-typedef std::vector<rw_lock_debug_t*> Infos;
-
-/** Get the thread debug info
-@param[in]	infos		The rw-lock mode owned by the threads
-@param[in]	lock		rw-lock to check
-@return the thread debug info or NULL if not found */
-void
-rw_lock_get_debug_info(const rw_lock_t* lock, Infos* infos)
-{
-	rw_lock_debug_t*	info = NULL;
-
-	ut_ad(rw_lock_validate(lock));
-
-	rw_lock_debug_mutex_enter();
-
-	for (info = UT_LIST_GET_FIRST(lock->debug_list);
-	     info != NULL;
-	     info = UT_LIST_GET_NEXT(list, info)) {
-
-		if (os_thread_eq(info->thread_id, os_thread_get_curr_id())) {
-
-			infos->push_back(info);
-		}
-	}
-
-	rw_lock_debug_mutex_exit();
-}
-
 /** Checks if the thread has locked the rw-lock in the specified mode, with
 the pass value == 0.
 @param[in]	lock		rw-lock
 @param[in]	flags		specify lock types with OR of the
 				rw_lock_flag_t values
 @return true if locked */
-bool
-rw_lock_own_flagged(
-	const rw_lock_t*	lock,
-	rw_lock_flags_t		flags)
+bool rw_lock_own_flagged(const rw_lock_t* lock, rw_lock_flags_t flags)
 {
-	Infos	infos;
+	ut_ad(rw_lock_validate(lock));
 
-	rw_lock_get_debug_info(lock, &infos);
+	rw_lock_debug_mutex_enter();
 
-	Infos::const_iterator	end = infos.end();
-
-	for (Infos::const_iterator it = infos.begin(); it != end; ++it) {
-
-		const rw_lock_debug_t*	info = *it;
-
-		if (info->pass) {
+	for (rw_lock_debug_t* info = UT_LIST_GET_FIRST(lock->debug_list);
+	     info != NULL;
+	     info = UT_LIST_GET_NEXT(list, info)) {
+		if (!os_thread_eq(info->thread_id, os_thread_get_curr_id())
+		    || info->pass) {
 			continue;
 		}
 
-		ut_ad(os_thread_eq(info->thread_id, os_thread_get_curr_id()));
-
 		switch (info->lock_type) {
 		case RW_LOCK_S:
-
-			if (flags & RW_LOCK_FLAG_S) {
-				return(true);
+			if (!(flags & RW_LOCK_FLAG_S)) {
+				continue;
 			}
 			break;
 
 		case RW_LOCK_X:
-
-			if (flags & RW_LOCK_FLAG_X) {
-				return(true);
+			if (!(flags & RW_LOCK_FLAG_X)) {
+				continue;
 			}
 			break;
 
 		case RW_LOCK_SX:
-
-			if (flags & RW_LOCK_FLAG_SX) {
-				return(true);
+			if (!(flags & RW_LOCK_FLAG_SX)) {
+				continue;
 			}
+			break;
 		}
+
+		rw_lock_debug_mutex_exit();
+		return true;
 	}
 
-	return(false);
+	rw_lock_debug_mutex_exit();
+	return false;
 }
 
 /***************************************************************//**
@@ -1227,28 +1193,31 @@ rw_lock_t::locked_from() const
 	the same thread can call X lock recursively. */
 
 	std::ostringstream	msg;
-	Infos			infos;
+	bool			written = false;
 
-	rw_lock_get_debug_info(this, &infos);
+	ut_ad(rw_lock_validate(this));
 
-	ulint			i = 0;
-	Infos::const_iterator	end = infos.end();
+	rw_lock_debug_mutex_enter();
 
-	for (Infos::const_iterator it = infos.begin(); it != end; ++it, ++i) {
+	for (rw_lock_debug_t* info = UT_LIST_GET_FIRST(debug_list);
+	     info != NULL;
+	     info = UT_LIST_GET_NEXT(list, info)) {
+		if (!os_thread_eq(info->thread_id, os_thread_get_curr_id())) {
+			continue;
+		}
 
-		const rw_lock_debug_t*	info = *it;
-
-		ut_ad(os_thread_eq(info->thread_id, os_thread_get_curr_id()));
-
-		if (i > 0) {
+		if (written) {
 			msg << ", ";
 		}
 
+		written = true;
+
 		msg << info->file_name << ":" << info->line;
 	}
 
-	return(msg.str());
+	rw_lock_debug_mutex_exit();
 
+	return(msg.str());
 }
 
 /** Print the rw-lock information.

From aab5c557cfdf4a155e5eab69d0979bcb4b26c710 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 3 Aug 2018 17:39:38 +0300
Subject: [PATCH 110/164] MDEV-16830 Crash in ALTER TABLE DROP FOREIGN KEY

ha_innobase::inplace_alter_table(): Do nothing if INNOBASE_ALTER_INSTANT
flags (such as DROP FOREIGN KEY) was present.

Also, use ALTER_OPTIONS instead of the alias ALTER_CHANGE_CREATE_OPTION.

This bug was caused by MDEV-11369, MDEV-13134 or related work.
---
 .../suite/innodb/r/instant_alter.result       | 59 ++++++++++++++++++-
 mysql-test/suite/innodb/t/instant_alter.test  | 21 +++++++
 storage/innobase/handler/handler0alter.cc     | 28 ++++-----
 3 files changed, 91 insertions(+), 17 deletions(-)

diff --git a/mysql-test/suite/innodb/r/instant_alter.result b/mysql-test/suite/innodb/r/instant_alter.result
index a70a3d077e0..bb15b665d49 100644
--- a/mysql-test/suite/innodb/r/instant_alter.result
+++ b/mysql-test/suite/innodb/r/instant_alter.result
@@ -446,6 +446,25 @@ ALTER TABLE t1 ADD COLUMN c INT;
 UPDATE t1 SET c = 1;
 UPDATE t1 SET c = 2;
 DROP TABLE t1;
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+CREATE TABLE t2 (b INT PRIMARY KEY, FOREIGN KEY(b) REFERENCES t1(a))
+ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+INSERT INTO t1 SET a=1;
+INSERT INTO t2 SET b=1;
+ALTER TABLE t2 ADD COLUMN a INT, DROP FOREIGN KEY t2_ibfk_1;
+ALTER TABLE t2 ADD INDEX(a);
+ALTER TABLE t1 ADD COLUMN b INT, ADD FOREIGN KEY(a) REFERENCES t2(a),
+ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Adding foreign keys needs foreign_key_checks=OFF. Try ALGORITHM=COPY
+SET foreign_key_checks=0;
+ALTER TABLE t1 ADD COLUMN b INT, ADD FOREIGN KEY(a) REFERENCES t2(a),
+ALGORITHM=INSTANT;
+ALTER TABLE t2 ADD CONSTRAINT fk FOREIGN KEY(b) REFERENCES t1(a),
+ALGORITHM=INSTANT;
+SET foreign_key_checks=1;
+ALTER TABLE t2 COMMENT 'domestic keys only', DROP FOREIGN KEY fk;
+ALTER TABLE t1 DROP FOREIGN KEY t1_ibfk_1;
+DROP TABLE t2, t1;
 CREATE TABLE t1
 (id INT PRIMARY KEY, c2 INT UNIQUE,
 c3 POINT NOT NULL DEFAULT ST_GeomFromText('POINT(3 4)'),
@@ -838,6 +857,25 @@ ALTER TABLE t1 ADD COLUMN c INT;
 UPDATE t1 SET c = 1;
 UPDATE t1 SET c = 2;
 DROP TABLE t1;
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB ROW_FORMAT=COMPACT;
+CREATE TABLE t2 (b INT PRIMARY KEY, FOREIGN KEY(b) REFERENCES t1(a))
+ENGINE=InnoDB ROW_FORMAT=COMPACT;
+INSERT INTO t1 SET a=1;
+INSERT INTO t2 SET b=1;
+ALTER TABLE t2 ADD COLUMN a INT, DROP FOREIGN KEY t2_ibfk_1;
+ALTER TABLE t2 ADD INDEX(a);
+ALTER TABLE t1 ADD COLUMN b INT, ADD FOREIGN KEY(a) REFERENCES t2(a),
+ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Adding foreign keys needs foreign_key_checks=OFF. Try ALGORITHM=COPY
+SET foreign_key_checks=0;
+ALTER TABLE t1 ADD COLUMN b INT, ADD FOREIGN KEY(a) REFERENCES t2(a),
+ALGORITHM=INSTANT;
+ALTER TABLE t2 ADD CONSTRAINT fk FOREIGN KEY(b) REFERENCES t1(a),
+ALGORITHM=INSTANT;
+SET foreign_key_checks=1;
+ALTER TABLE t2 COMMENT 'domestic keys only', DROP FOREIGN KEY fk;
+ALTER TABLE t1 DROP FOREIGN KEY t1_ibfk_1;
+DROP TABLE t2, t1;
 CREATE TABLE t1
 (id INT PRIMARY KEY, c2 INT UNIQUE,
 c3 POINT NOT NULL DEFAULT ST_GeomFromText('POINT(3 4)'),
@@ -1230,10 +1268,29 @@ ALTER TABLE t1 ADD COLUMN c INT;
 UPDATE t1 SET c = 1;
 UPDATE t1 SET c = 2;
 DROP TABLE t1;
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+CREATE TABLE t2 (b INT PRIMARY KEY, FOREIGN KEY(b) REFERENCES t1(a))
+ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+INSERT INTO t1 SET a=1;
+INSERT INTO t2 SET b=1;
+ALTER TABLE t2 ADD COLUMN a INT, DROP FOREIGN KEY t2_ibfk_1;
+ALTER TABLE t2 ADD INDEX(a);
+ALTER TABLE t1 ADD COLUMN b INT, ADD FOREIGN KEY(a) REFERENCES t2(a),
+ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Adding foreign keys needs foreign_key_checks=OFF. Try ALGORITHM=COPY
+SET foreign_key_checks=0;
+ALTER TABLE t1 ADD COLUMN b INT, ADD FOREIGN KEY(a) REFERENCES t2(a),
+ALGORITHM=INSTANT;
+ALTER TABLE t2 ADD CONSTRAINT fk FOREIGN KEY(b) REFERENCES t1(a),
+ALGORITHM=INSTANT;
+SET foreign_key_checks=1;
+ALTER TABLE t2 COMMENT 'domestic keys only', DROP FOREIGN KEY fk;
+ALTER TABLE t1 DROP FOREIGN KEY t1_ibfk_1;
+DROP TABLE t2, t1;
 disconnect analyze;
 SELECT variable_value-@old_instant instants
 FROM information_schema.global_status
 WHERE variable_name = 'innodb_instant_alter_column';
 instants
-39
+45
 SET GLOBAL innodb_purge_rseg_truncate_frequency= @saved_frequency;
diff --git a/mysql-test/suite/innodb/t/instant_alter.test b/mysql-test/suite/innodb/t/instant_alter.test
index c414b92f713..18c07bdac73 100644
--- a/mysql-test/suite/innodb/t/instant_alter.test
+++ b/mysql-test/suite/innodb/t/instant_alter.test
@@ -319,6 +319,27 @@ UPDATE t1 SET c = 1;
 UPDATE t1 SET c = 2;
 DROP TABLE t1;
 
+# MDEV-16830 Crash in ALTER TABLE DROP FOREIGN KEY
+eval CREATE TABLE t1 (a INT PRIMARY KEY) $engine;
+eval CREATE TABLE t2 (b INT PRIMARY KEY, FOREIGN KEY(b) REFERENCES t1(a))
+$engine;
+INSERT INTO t1 SET a=1;
+INSERT INTO t2 SET b=1;
+ALTER TABLE t2 ADD COLUMN a INT, DROP FOREIGN KEY t2_ibfk_1;
+ALTER TABLE t2 ADD INDEX(a);
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
+ALTER TABLE t1 ADD COLUMN b INT, ADD FOREIGN KEY(a) REFERENCES t2(a),
+ALGORITHM=INSTANT;
+SET foreign_key_checks=0;
+ALTER TABLE t1 ADD COLUMN b INT, ADD FOREIGN KEY(a) REFERENCES t2(a),
+ALGORITHM=INSTANT;
+ALTER TABLE t2 ADD CONSTRAINT fk FOREIGN KEY(b) REFERENCES t1(a),
+ALGORITHM=INSTANT;
+SET foreign_key_checks=1;
+ALTER TABLE t2 COMMENT 'domestic keys only', DROP FOREIGN KEY fk;
+ALTER TABLE t1 DROP FOREIGN KEY t1_ibfk_1;
+DROP TABLE t2, t1;
+
 dec $format;
 }
 disconnect analyze;
diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc
index a8cbee11c8e..9c6791a3e28 100644
--- a/storage/innobase/handler/handler0alter.cc
+++ b/storage/innobase/handler/handler0alter.cc
@@ -77,8 +77,8 @@ static const alter_table_operations INNOBASE_DEFAULTS
 static const alter_table_operations INNOBASE_ALTER_REBUILD
 	= ALTER_ADD_PK_INDEX
 	| ALTER_DROP_PK_INDEX
-	| ALTER_CHANGE_CREATE_OPTION
-	/* CHANGE_CREATE_OPTION needs to check create_option_need_rebuild() */
+	| ALTER_OPTIONS
+	/* ALTER_OPTIONS needs to check create_option_need_rebuild() */
 	| ALTER_COLUMN_NULLABLE
 	| INNOBASE_DEFAULTS
 	| ALTER_STORED_COLUMN_ORDER
@@ -498,8 +498,7 @@ static bool create_option_need_rebuild(
 	const Alter_inplace_info*	ha_alter_info,
 	const TABLE*			table)
 {
-	DBUG_ASSERT(ha_alter_info->handler_flags
-		    & ALTER_CHANGE_CREATE_OPTION);
+	DBUG_ASSERT(ha_alter_info->handler_flags & ALTER_OPTIONS);
 
 	if (ha_alter_info->create_info->used_fields
 	    & (HA_CREATE_USED_ROW_FORMAT
@@ -541,7 +540,7 @@ innobase_need_rebuild(
 {
 	if ((ha_alter_info->handler_flags & ~(INNOBASE_INPLACE_IGNORE
 					      | INNOBASE_ALTER_INSTANT))
-	    == ALTER_CHANGE_CREATE_OPTION) {
+	    == ALTER_OPTIONS) {
 		return create_option_need_rebuild(ha_alter_info, table);
 	}
 
@@ -662,8 +661,7 @@ instant_alter_column_possible(
 		return false;
 	}
 
-	if (~ha_alter_info->handler_flags
-	    & ALTER_ADD_STORED_BASE_COLUMN) {
+	if (~ha_alter_info->handler_flags & ALTER_ADD_STORED_BASE_COLUMN) {
 		return false;
 	}
 
@@ -686,13 +684,11 @@ instant_alter_column_possible(
 	columns. */
 	if (ha_alter_info->handler_flags
 	    & ((INNOBASE_ALTER_REBUILD | INNOBASE_ONLINE_CREATE)
-	       & ~ALTER_ADD_STORED_BASE_COLUMN
-	       & ~ALTER_CHANGE_CREATE_OPTION)) {
+	       & ~ALTER_ADD_STORED_BASE_COLUMN & ~ALTER_OPTIONS)) {
 		return false;
 	}
 
-	return !(ha_alter_info->handler_flags
-		 & ALTER_CHANGE_CREATE_OPTION)
+	return !(ha_alter_info->handler_flags & ALTER_OPTIONS)
 		|| !create_option_need_rebuild(ha_alter_info, table);
 }
 
@@ -5457,8 +5453,7 @@ not_instant_add_column:
 			}
 		}
 
-		if (ha_alter_info->handler_flags
-		    & ALTER_CHANGE_CREATE_OPTION) {
+		if (ha_alter_info->handler_flags & ALTER_OPTIONS) {
 			const ha_table_option_struct& alt_opt=
 				*ha_alter_info->create_info->option_struct;
 			const ha_table_option_struct& opt=
@@ -6827,7 +6822,7 @@ err_exit:
 	if (!(ha_alter_info->handler_flags & INNOBASE_ALTER_DATA)
 	    || ((ha_alter_info->handler_flags & ~(INNOBASE_INPLACE_IGNORE
 						  | INNOBASE_ALTER_INSTANT))
-		== ALTER_CHANGE_CREATE_OPTION
+		== ALTER_OPTIONS
 		&& !create_option_need_rebuild(ha_alter_info, table))) {
 
 		if (heap) {
@@ -7107,8 +7102,9 @@ ok_exit:
 		DBUG_RETURN(false);
 	}
 
-	if ((ha_alter_info->handler_flags & ~INNOBASE_INPLACE_IGNORE)
-	    == ALTER_CHANGE_CREATE_OPTION
+	if ((ha_alter_info->handler_flags & ~(INNOBASE_INPLACE_IGNORE
+					      | INNOBASE_ALTER_INSTANT))
+	    == ALTER_OPTIONS
 	    && !create_option_need_rebuild(ha_alter_info, table)) {
 		goto ok_exit;
 	}

From 7749745b7d688bd35d92219869f93dd0b44117fc Mon Sep 17 00:00:00 2001
From: Oleksandr Byelkin <sanja@mariadb.com>
Date: Fri, 3 Aug 2018 22:39:57 +0200
Subject: [PATCH 111/164] Fix of type, to make windows compiler happy.

---
 sql/item_strfunc.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc
index 1590576ddf4..51723a25dee 100644
--- a/sql/item_strfunc.cc
+++ b/sql/item_strfunc.cc
@@ -1570,7 +1570,7 @@ String *Item_str_conv::val_str(String *str)
 {
   DBUG_ASSERT(fixed == 1);
   String *res;
-  uint32 alloced_length, len;
+  size_t alloced_length, len;
 
   if ((null_value= (!(res= args[0]->val_str(&tmp_value)) ||
                     str->alloc((alloced_length= res->length() * multiply)))))

From d453374fc480112266996a1026b97654cc174c09 Mon Sep 17 00:00:00 2001
From: Igor Babaev <igor@askmonty.org>
Date: Fri, 3 Aug 2018 14:54:02 -0700
Subject: [PATCH 112/164] MDEV-16801 seg_fault on a query

The bug was in the in the code of JOIN::check_for_splittable_materialized()
where the structures describing the fields of a materialized derived
table that potentially could be used in split optimization were build.
As a result of this bug some fields that were not usable for splitting
were detected as usable. This could trigger crashes further in
st_join_table::choose_best_splitting().
---
 mysql-test/main/derived_cond_pushdown.result | 54 ++++++++++++++++++++
 mysql-test/main/derived_cond_pushdown.test   | 33 ++++++++++++
 sql/opt_split.cc                             |  3 +-
 3 files changed, 89 insertions(+), 1 deletion(-)

diff --git a/mysql-test/main/derived_cond_pushdown.result b/mysql-test/main/derived_cond_pushdown.result
index a2268ced84f..832b9a46289 100644
--- a/mysql-test/main/derived_cond_pushdown.result
+++ b/mysql-test/main/derived_cond_pushdown.result
@@ -16026,3 +16026,57 @@ SELECT a FROM (SELECT "aa" a) t WHERE f1(t.a, (SELECT MAX('aa') FROM DUAL LIMIT
 a
 aa
 DROP FUNCTION f1;
+#
+# MDEV-16801: splittable materialized derived/views with
+#             one grouping field from table without keys
+#
+CREATE TABLE t1 (a int, b int, INDEX idx_a(a), INDEX idx_b(b)) ENGINE=MYISAM;
+CREATE TABLE t2 (c int) ENGINE=MYISAM;
+CREATE TABLE t3 (d int) ENGINE=MYISAM;
+INSERT INTO t1 VALUES
+(77,7), (11,1), (33,3), (44,4), (8,88),
+(78,7), (98,9), (38,3), (28,2), (79,7),
+(58,5), (42,4), (71,7), (27,2), (91,9);
+INSERT INTO t1 SELECT a+100, b+10 FROM t1;
+INSERT INTO t2 VALUES
+(100), (700), (200), (100), (200);
+INSERT INTO t3 VALUES
+(3), (4), (1), (8), (3);
+ANALYZE tables t1,t2,t3;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	OK
+test.t2	analyze	status	OK
+test.t3	analyze	status	OK
+SELECT *
+FROM t3,
+(SELECT t1.b, t2.c
+FROM  t1, t2
+GROUP BY t1.b,t2.c) dt
+WHERE t3.d = dt.b;
+d	b	c
+3	3	700
+3	3	200
+3	3	100
+4	4	700
+4	4	200
+4	4	100
+1	1	700
+1	1	200
+1	1	100
+3	3	700
+3	3	200
+3	3	100
+EXPLAIN EXTENDED SELECT *
+FROM t3,
+(SELECT t1.b, t2.c
+FROM  t1, t2
+GROUP BY t1.b,t2.c) dt
+WHERE t3.d = dt.b;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	PRIMARY	t3	ALL	NULL	NULL	NULL	NULL	5	100.00	Using where
+1	PRIMARY	<derived2>	ref	key0	key0	5	test.t3.d	2	100.00	
+2	LATERAL DERIVED	t1	ref	idx_b	idx_b	5	test.t3.d	2	100.00	Using index; Using temporary; Using filesort
+2	LATERAL DERIVED	t2	ALL	NULL	NULL	NULL	NULL	5	100.00	Using join buffer (flat, BNL join)
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t3`.`d` AS `d`,`dt`.`b` AS `b`,`dt`.`c` AS `c` from `test`.`t3` join (/* select#2 */ select `test`.`t1`.`b` AS `b`,`test`.`t2`.`c` AS `c` from `test`.`t1` join `test`.`t2` where `test`.`t1`.`b` = `test`.`t3`.`d` group by `test`.`t1`.`b`,`test`.`t2`.`c`) `dt` where `dt`.`b` = `test`.`t3`.`d`
+DROP TABLE t1,t2,t3;
diff --git a/mysql-test/main/derived_cond_pushdown.test b/mysql-test/main/derived_cond_pushdown.test
index 0c3a542f907..b75c56e9ff2 100644
--- a/mysql-test/main/derived_cond_pushdown.test
+++ b/mysql-test/main/derived_cond_pushdown.test
@@ -2986,3 +2986,36 @@ END;$$
 DELIMITER ;$$
 SELECT a FROM (SELECT "aa" a) t WHERE f1(t.a, (SELECT MAX('aa') FROM DUAL LIMIT 1));
 DROP FUNCTION f1;
+
+--echo #
+--echo # MDEV-16801: splittable materialized derived/views with
+--echo #             one grouping field from table without keys
+--echo #
+
+CREATE TABLE t1 (a int, b int, INDEX idx_a(a), INDEX idx_b(b)) ENGINE=MYISAM;
+CREATE TABLE t2 (c int) ENGINE=MYISAM;
+CREATE TABLE t3 (d int) ENGINE=MYISAM;
+INSERT INTO t1 VALUES
+  (77,7), (11,1), (33,3), (44,4), (8,88),
+  (78,7), (98,9), (38,3), (28,2), (79,7),
+  (58,5), (42,4), (71,7), (27,2), (91,9);
+INSERT INTO t1 SELECT a+100, b+10 FROM t1;
+INSERT INTO t2 VALUES
+  (100), (700), (200), (100), (200);
+INSERT INTO t3 VALUES
+  (3), (4), (1), (8), (3);
+
+ANALYZE tables t1,t2,t3;
+
+let $q=
+SELECT *
+  FROM t3,
+       (SELECT t1.b, t2.c
+          FROM  t1, t2
+            GROUP BY t1.b,t2.c) dt
+WHERE t3.d = dt.b;
+
+eval $q;
+eval EXPLAIN EXTENDED $q;
+
+DROP TABLE t1,t2,t3;
diff --git a/sql/opt_split.cc b/sql/opt_split.cc
index 37853bdbbe9..611e70376c8 100644
--- a/sql/opt_split.cc
+++ b/sql/opt_split.cc
@@ -413,7 +413,8 @@ bool JOIN::check_for_splittable_materialized()
 
         for (cand= cand_start; cand < cand_end; cand++)
         {
-          if (cand->underlying_field->field_index + 1 == fldnr)
+          if (cand->underlying_field->table == table &&
+              cand->underlying_field->field_index + 1 == fldnr)
 	  {
             cand->is_usable_for_ref_access= true;
             break;

From f4e2db5b434516c03c0b29f73a07c3041c36cdcf Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Sun, 5 Aug 2018 00:33:12 +0100
Subject: [PATCH 113/164] MDEV-16544 - crash in ha_sphinx::create()

Use table_arg that was passed to the function, instead of dereferencing
this->table, which is a NULL pointer.
---
 storage/sphinx/ha_sphinx.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/storage/sphinx/ha_sphinx.cc b/storage/sphinx/ha_sphinx.cc
index 580b3fe539f..90409213843 100644
--- a/storage/sphinx/ha_sphinx.cc
+++ b/storage/sphinx/ha_sphinx.cc
@@ -3441,7 +3441,7 @@ int ha_sphinx::create ( const char * name, TABLE * table_arg, HA_CREATE_INFO * )
 		if (
 			table_arg->s->keys!=1 ||
 			table_arg->key_info[0].user_defined_key_parts!=1 ||
-			strcasecmp ( table_arg->key_info[0].key_part[0].field->field_name.str, table->field[2]->field_name.str ) )
+			strcasecmp ( table_arg->key_info[0].key_part[0].field->field_name.str, table_arg->field[2]->field_name.str ) )
 		{
 			my_snprintf ( sError, sizeof(sError), "%s: there must be an index on '%s' column",
 				name, table_arg->field[2]->field_name.str );

From 3570ea9189183705f639e41f7828055bee60ee43 Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Sun, 5 Aug 2018 00:36:59 +0100
Subject: [PATCH 114/164] remove warning on Windows

Replace do-it-yourself version of strdup() with real strdup().
---
 extra/mariabackup/backup_copy.cc | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc
index 8b474c37e4b..907e5e94257 100644
--- a/extra/mariabackup/backup_copy.cc
+++ b/extra/mariabackup/backup_copy.cc
@@ -644,13 +644,10 @@ int
 mkdirp(const char *pathname, int Flags, myf MyFlags)
 {
 	char *parent, *p;
-	int len = strlen(pathname) + 1;
 
 	/* make a parent directory path */
-	if (!(parent= (char *)malloc(len)))
+	if (!(parent= strdup(pathname)))
           return(-1);
-	strncpy(parent, pathname, len);
-	parent[len-1]= 0;
 
 	for (p = parent + strlen(parent);
 	    !is_path_separator(*p) && p != parent; p--);

From cb1945dd0dd244b0fb901c1b950b4ef856c1b725 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= <otto@mariadb.org>
Date: Sun, 5 Aug 2018 17:11:15 +0300
Subject: [PATCH 115/164] MDEV-16666: Partially revert "Deb: Update
 documentation and fix spelling errors"

This partially reverts commit 548ec3a088a2444881e1245a237f81cea1671969
by removing the misfixed misspellings.
---
 debian/additions/innotop/changelog.innotop |  2 +-
 debian/additions/innotop/innotop           | 14 +++++++-------
 debian/additions/innotop/innotop.1         |  6 +++---
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/debian/additions/innotop/changelog.innotop b/debian/additions/innotop/changelog.innotop
index 67bc52698e7..1816595240b 100644
--- a/debian/additions/innotop/changelog.innotop
+++ b/debian/additions/innotop/changelog.innotop
@@ -77,7 +77,7 @@ Changelog for innotop:
    * remove cxn from $meta->{group_by} if there's only one connection displayed
    * fix for issue 19 - cxn column won't become visible when viewing two 
      connections after having viewed one connection
-   * supress errors resulting from the addition of a 'BACKGROUND THREAD' 
+   * suppress errors resulting from the addition of a 'BACKGROUND THREAD' 
      section in the output of 'show innodb status' 
    * possible fix for issue 22 - Useless use of a constant in void context
    * small change to set_to_tbl() around hiding the cxn column if there 
diff --git a/debian/additions/innotop/innotop b/debian/additions/innotop/innotop
index f958eba432b..b0134e0bc91 100644
--- a/debian/additions/innotop/innotop
+++ b/debian/additions/innotop/innotop
@@ -17,7 +17,7 @@
 
 # You should have received a copy of the GNU General Public License along with
 # this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
-# Street, Fifth Floor, Boston, MA  02110-1301  USA
+# Street, Fifth Floor, Boston, MA 02111-1301 USA
 
 use strict;
 use warnings FATAL => 'all';
@@ -1438,8 +1438,8 @@ systems, you can issue `man perlgpl' or `man perlartistic' to read these
 licenses.
 
 You should have received a copy of the GNU General Public License along with
-this program; if not, write to the Free Software Foundation, Inc., 59 Temple
-Place, Suite 330, Boston, MA  02111-1307  USA.
+this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
+Street, Fifth Floor, Boston, MA 02111-1301 USA.
 LICENSE
 
 # Configuration information and global setup {{{1
@@ -4657,7 +4657,7 @@ my %stmt_maker_for = (
       my $sth;
       eval { # This can fail if the table doesn't exist, INFORMATION_SCHEMA doesn't exist, etc.
          my $cols = $dbh->selectall_arrayref(q{SHOW /*innotop*/ COLUMNS FROM INFORMATION_SCHEMA.PROCESSLIST LIKE 'TIME_MS'});
-         if ( @$cols ) { # The TIME_MS colum exists
+         if ( @$cols ) { # The TIME_MS column exists
             $sth = $dbh->prepare(q{SELECT /*innotop*/ ID, USER, HOST, DB, COMMAND, CASE WHEN TIME_MS/1000 > 365*86400 THEN TIME ELSE TIME_MS/1000 END AS TIME, STATE, INFO FROM INFORMATION_SCHEMA.PROCESSLIST});
          }
       };
@@ -11653,7 +11653,7 @@ show you something like this:
  pages_modified     Dirty Pages  Pages modified (dirty  IB_bp_pages_m
  buf_pool_hit_rate  Hit Rate     Buffer pool hit rate   IB_bp_buf_poo
  total_mem_alloc    Memory       Total memory allocate  IB_bp_total_m
- add_pool_alloc     Add'l Pool   Additonal pool alloca  IB_bp_add_poo
+ add_pool_alloc     Add'l Pool   Additional pool alloca  IB_bp_add_poo
 
 The first line shows which table you're editing, and reminds you again to press
 '?' for a list of key mappings.  The rest is a tabular representation of the
@@ -12233,8 +12233,8 @@ systems, you can issue `man perlgpl' or `man perlartistic' to read these
 licenses.
 
 You should have received a copy of the GNU General Public License along with
-this program; if not, write to the Free Software Foundation, Inc., 59 Temple
-Place, Suite 330, Boston, MA  02111-1307  USA.
+this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
+Street, Fifth Floor, Boston, MA 02111-1301 USA.
 
 Execute innotop and press '!' to see this information at any time.
 
diff --git a/debian/additions/innotop/innotop.1 b/debian/additions/innotop/innotop.1
index 7cd275a79f2..7079ff2c4a4 100644
--- a/debian/additions/innotop/innotop.1
+++ b/debian/additions/innotop/innotop.1
@@ -1678,7 +1678,7 @@ show you something like this:
 \& pages_modified     Dirty Pages  Pages modified (dirty  IB_bp_pages_m
 \& buf_pool_hit_rate  Hit Rate     Buffer pool hit rate   IB_bp_buf_poo
 \& total_mem_alloc    Memory       Total memory allocate  IB_bp_total_m
-\& add_pool_alloc     Add\*(Aql Pool   Additonal pool alloca  IB_bp_add_poo
+\& add_pool_alloc     Add\*(Aql Pool   Additional pool alloca  IB_bp_add_poo
 .Ve
 .PP
 The first line shows which table you're editing, and reminds you again to press
@@ -2183,8 +2183,8 @@ systems, you can issue `man perlgpl' or `man perlartistic' to read these
 licenses.
 .PP
 You should have received a copy of the \s-1GNU\s0 General Public License along with
-this program; if not, write to the Free Software Foundation, Inc., 59 Temple
-Place, Suite 330, Boston, \s-1MA  02111\-1307  USA.\s0
+this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
+Street, Fifth Floor, Boston, \s-1MA\s0  02111\-1301  \s-1USA\s0.
 .PP
 Execute innotop and press '!' to see this information at any time.
 .SH "AUTHOR"

From fc324a5f87827e43f51d1a75e60b3d5abcd807e5 Mon Sep 17 00:00:00 2001
From: Alexey Botchkov <holyfoot@askmonty.org>
Date: Sun, 5 Aug 2018 18:33:17 +0400
Subject: [PATCH 116/164] MDEV-16869 String functions don't respect character
 set of JSON_VALUE.

Item_func_json_value::val_str() produced string of wrong charset.
---
 mysql-test/r/func_json.result | 12 ++++++++++++
 mysql-test/t/func_json.test   | 11 +++++++++++
 sql/item_jsonfunc.cc          |  2 +-
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/mysql-test/r/func_json.result b/mysql-test/r/func_json.result
index 206f745a0d2..2a2df00ff61 100644
--- a/mysql-test/r/func_json.result
+++ b/mysql-test/r/func_json.result
@@ -766,3 +766,15 @@ DROP TABLE t1;
 select json_array(1,uuid(),compress(5.140264e+307));
 json_array(1,uuid(),compress(5.140264e+307))
 NULL
+#
+# MDEV-16869 String functions don't respect character set of JSON_VALUE.
+#
+create table t1(json_col TEXT) DEFAULT CHARSET=latin1;
+insert into t1 values (_latin1 X'7B226B657931223A2253EC227D');
+select JSON_VALUE(json_col, '$.key1')= _latin1 X'53EC' from t1;
+JSON_VALUE(json_col, '$.key1')= _latin1 X'53EC'
+1
+select REPLACE(JSON_VALUE(json_col, '$.key1'), 'null', '') = _latin1 X'53EC' from t1;
+REPLACE(JSON_VALUE(json_col, '$.key1'), 'null', '') = _latin1 X'53EC'
+1
+drop table t1;
diff --git a/mysql-test/t/func_json.test b/mysql-test/t/func_json.test
index c2cc00dd6e2..41e520404b6 100644
--- a/mysql-test/t/func_json.test
+++ b/mysql-test/t/func_json.test
@@ -429,3 +429,14 @@ DROP TABLE t1;
 
 select json_array(1,uuid(),compress(5.140264e+307));
 
+
+--echo #
+--echo # MDEV-16869 String functions don't respect character set of JSON_VALUE.
+--echo #
+
+create table t1(json_col TEXT) DEFAULT CHARSET=latin1;
+insert into t1 values (_latin1 X'7B226B657931223A2253EC227D');
+select JSON_VALUE(json_col, '$.key1')= _latin1 X'53EC' from t1;
+select REPLACE(JSON_VALUE(json_col, '$.key1'), 'null', '') = _latin1 X'53EC' from t1;
+drop table t1;
+
diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc
index ac0d7e9e6b0..e9d38b78843 100644
--- a/sql/item_jsonfunc.cc
+++ b/sql/item_jsonfunc.cc
@@ -479,7 +479,7 @@ String *Item_func_json_value::val_str(String *str)
                   (const uchar *) js->ptr() + js->length());
 
   str->length(0);
-  str->set_charset(&my_charset_utf8mb4_bin);
+  str->set_charset(collation.collation);
 
   path.cur_step= path.p.steps;
 continue_search:

From 4ddcb4eb46c62cf459936554d43351db740ba14d Mon Sep 17 00:00:00 2001
From: Alexey Botchkov <holyfoot@askmonty.org>
Date: Mon, 6 Aug 2018 13:37:09 +0400
Subject: [PATCH 117/164] MDEV-16750 JSON_SET mishandles unicode every second
 pair of arguments.

The charset of temporary storage (Item_func_json_insert::tmp_js)
  was not properly set.
---
 mysql-test/r/func_json.result | 12 ++++++++++++
 mysql-test/t/func_json.test   |  8 ++++++++
 sql/item_jsonfunc.cc          |  6 +++---
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/mysql-test/r/func_json.result b/mysql-test/r/func_json.result
index 2a2df00ff61..e8b2fac4fb8 100644
--- a/mysql-test/r/func_json.result
+++ b/mysql-test/r/func_json.result
@@ -778,3 +778,15 @@ select REPLACE(JSON_VALUE(json_col, '$.key1'), 'null', '') = _latin1 X'53EC' fro
 REPLACE(JSON_VALUE(json_col, '$.key1'), 'null', '') = _latin1 X'53EC'
 1
 drop table t1;
+#
+# MDEV-16750 JSON_SET mishandles unicode every second pair of arguments.
+#
+SELECT JSON_SET('{}', '$.a', _utf8 0xC3B6);
+JSON_SET('{}', '$.a', _utf8 0xC3B6)
+{"a": "ö"}
+SELECT JSON_SET('{}', '$.a', _utf8 0xC3B6, '$.b', _utf8 0xC3B6);
+JSON_SET('{}', '$.a', _utf8 0xC3B6, '$.b', _utf8 0xC3B6)
+{"a": "ö", "b": "ö"}
+SELECT JSON_SET('{}', '$.a', _utf8 X'C3B6', '$.x', 1, '$.b', _utf8 X'C3B6');
+JSON_SET('{}', '$.a', _utf8 X'C3B6', '$.x', 1, '$.b', _utf8 X'C3B6')
+{"a": "ö", "x": 1, "b": "ö"}
diff --git a/mysql-test/t/func_json.test b/mysql-test/t/func_json.test
index 41e520404b6..fd5e3d7d7ee 100644
--- a/mysql-test/t/func_json.test
+++ b/mysql-test/t/func_json.test
@@ -440,3 +440,11 @@ select JSON_VALUE(json_col, '$.key1')= _latin1 X'53EC' from t1;
 select REPLACE(JSON_VALUE(json_col, '$.key1'), 'null', '') = _latin1 X'53EC' from t1;
 drop table t1;
 
+--echo #
+--echo # MDEV-16750 JSON_SET mishandles unicode every second pair of arguments.
+--echo #
+
+SELECT JSON_SET('{}', '$.a', _utf8 0xC3B6);
+SELECT JSON_SET('{}', '$.a', _utf8 0xC3B6, '$.b', _utf8 0xC3B6);
+SELECT JSON_SET('{}', '$.a', _utf8 X'C3B6', '$.x', 1, '$.b', _utf8 X'C3B6');
+
diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc
index e9d38b78843..4a837646451 100644
--- a/sql/item_jsonfunc.cc
+++ b/sql/item_jsonfunc.cc
@@ -2384,8 +2384,9 @@ String *Item_func_json_insert::val_str(String *str)
   if ((null_value= args[0]->null_value))
     return 0;
 
-  str->set_charset(js->charset());
-  json_string_set_cs(&key_name, js->charset());
+  str->set_charset(collation.collation);
+  tmp_js.set_charset(collation.collation);
+  json_string_set_cs(&key_name, collation.collation);
 
   for (n_arg=1, n_path=0; n_arg < arg_count; n_arg+=2, n_path++)
   {
@@ -2599,7 +2600,6 @@ continue_point:
   json_scan_start(&je, js->charset(),(const uchar *) js->ptr(),
                   (const uchar *) js->ptr() + js->length());
   str->length(0);
-  str->set_charset(js->charset());
   if (json_nice(&je, str, Item_func_json_format::LOOSE))
     goto js_error;
 

From 9644415fa9d2abf3f8e2265570dc7fbca046b59c Mon Sep 17 00:00:00 2001
From: Olivier Bertrand <bertrandop@gmail.com>
Date: Mon, 6 Aug 2018 19:42:00 +0200
Subject: [PATCH 118/164] - Fix MDEV-16672 Connect: Warnings with 10.0  
 filamtxt.cpp: DOSFAM::RenameTempFile: Change sprintf to snprintf.  
 filamvct.cpp: VECFAM::RenameTempFile: Change sprintf to snprintf.  
 javaconn.cpp:     Add JAVAConn::GetUTFString function.     Use it instead of
 env->GetStringUTFChars.     Fix wrong identation.   javaconn.h: Add
 GetUTFString declaration.   jdbconn.cpp:     Use GetUTFString function
 instead of env->GetStringUTFChars.   jmgoconn.cpp:     Use GetUTFString
 function instead of env->GetStringUTFChars.     Fix wrong identation.  
 jsonudf.cpp: change 139 to BMX line 4631.   tabjmg.cpp:     Add
 ReleaseStringUTF.     Fix wrong identation.   tabpivot.cpp: Fix wrong
 identation.   tabutil.cpp: TDBPRX::GetSubTable: Change sprintf to snprintf.  
 modified:   storage/connect/filamtxt.cpp   modified:  
 storage/connect/filamvct.cpp   modified:   storage/connect/javaconn.cpp  
 modified:   storage/connect/javaconn.h   modified:  
 storage/connect/jdbconn.cpp   modified:   storage/connect/jmgoconn.cpp  
 modified:   storage/connect/jsonudf.cpp   modified:  
 storage/connect/tabjmg.cpp   modified:   storage/connect/tabpivot.cpp  
 modified:   storage/connect/tabutil.cpp

- Fix MDEV-16895 CONNECT engine's get_error_message can cause buffer
                 overflow and server crash with long queries
  ha_connect_cc: Update version.
    get_error_message: Remove charset conversion.
  modified:   storage/connect/ha_connect.cc

- Fix a server crash on inserting bigint to a JDBC table
  JDBConn::SetUUID:
    Suppress check on ctyp that causes a server crash because ctyp
    can be negative and this triggers an DEBUG_ASSERT on return.
  modified:   storage/connect/jdbconn.cpp

- Update jdbc.result
  mysql-test/connect/r/jdbc.result: Recorded to reflect a message change.
  modified:   storage/connect/mysql-test/connect/r/jdbc.result
---
 storage/connect/filamtxt.cpp                  |   4 +-
 storage/connect/filamvct.cpp                  |   6 +-
 storage/connect/ha_connect.cc                 |  24 +--
 storage/connect/javaconn.cpp                  |  18 +-
 storage/connect/javaconn.h                    |   1 +
 storage/connect/jdbconn.cpp                   |  36 ++--
 storage/connect/jmgoconn.cpp                  |   6 +-
 storage/connect/jsonudf.cpp                   |   2 +-
 .../connect/mysql-test/connect/r/jdbc.result  |   3 +-
 storage/connect/tabjmg.cpp                    |  11 +-
 storage/connect/tabpivot.cpp                  | 180 +++++++++---------
 storage/connect/tabutil.cpp                   |   2 +-
 12 files changed, 144 insertions(+), 149 deletions(-)

diff --git a/storage/connect/filamtxt.cpp b/storage/connect/filamtxt.cpp
index 490ca3a5fba..ca48fc765a1 100644
--- a/storage/connect/filamtxt.cpp
+++ b/storage/connect/filamtxt.cpp
@@ -1173,11 +1173,11 @@ int DOSFAM::RenameTempFile(PGLOBAL g)
     remove(filetemp);   // May still be there from previous error
 
     if (rename(filename, filetemp)) {    // Save file for security
-      sprintf(g->Message, MSG(RENAME_ERROR),
+      snprintf(g->Message, MAX_STR, MSG(RENAME_ERROR),
               filename, filetemp, strerror(errno));
 			throw 51;
 		} else if (rename(tempname, filename)) {
-      sprintf(g->Message, MSG(RENAME_ERROR),
+      snprintf(g->Message, MAX_STR, MSG(RENAME_ERROR),
               tempname, filename, strerror(errno));
       rc = rename(filetemp, filename);   // Restore saved file
 			throw 52;
diff --git a/storage/connect/filamvct.cpp b/storage/connect/filamvct.cpp
index a660461e9ee..e8a4ba890bf 100755
--- a/storage/connect/filamvct.cpp
+++ b/storage/connect/filamvct.cpp
@@ -353,7 +353,7 @@ int VCTFAM::Cardinality(PGLOBAL g)
 
     } // endif split
 
-    return (Block) ? ((Block - 1) * Nrec + Last) : 0;
+  return (Block) ? ((Block - 1) * Nrec + Last) : 0;
   } // end of Cardinality
 
 /***********************************************************************/
@@ -2458,11 +2458,11 @@ int VECFAM::RenameTempFile(PGLOBAL g)
       remove(filetemp);   // May still be there from previous error
   
       if (rename(filename, filetemp)) {    // Save file for security
-        sprintf(g->Message, MSG(RENAME_ERROR),
+        snprintf(g->Message, MAX_STR, MSG(RENAME_ERROR),
                 filename, filetemp, strerror(errno));
         rc = RC_FX;
       } else if (rename(tempname, filename)) {
-        sprintf(g->Message, MSG(RENAME_ERROR),
+        snprintf(g->Message, MAX_STR, MSG(RENAME_ERROR),
                 tempname, filename, strerror(errno));
         rc = rename(filetemp, filename);   // Restore saved file
         rc = RC_FX;
diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc
index e09eddc4555..8811030e751 100644
--- a/storage/connect/ha_connect.cc
+++ b/storage/connect/ha_connect.cc
@@ -170,7 +170,7 @@
 #define JSONMAX      10             // JSON Default max grp size
 
 extern "C" {
-       char version[]= "Version 1.06.0007 March 11, 2018";
+       char version[]= "Version 1.06.0007 August 06, 2018";
 #if defined(__WIN__)
        char compver[]= "Version 1.06.0007 " __DATE__ " "  __TIME__;
        char slash= '\\';
@@ -3303,23 +3303,15 @@ bool ha_connect::get_error_message(int error, String* buf)
 {
   DBUG_ENTER("ha_connect::get_error_message");
 
-  if (xp && xp->g) {
-    PGLOBAL g= xp->g;
-    char    msg[3072];         // MAX_STR * 3
-    uint    dummy_errors;
-    uint32  len= copy_and_convert(msg, strlen(g->Message) * 3,
-                               system_charset_info,
-                               g->Message, strlen(g->Message),
-                               &my_charset_latin1,
-                               &dummy_errors);
+	if (xp && xp->g) {
+		PGLOBAL g = xp->g;
 
-    if (trace(1))
-      htrc("GEM(%d): len=%u %s\n", error, len, g->Message);
+		if (trace(1))
+			htrc("GEM(%d): %s\n", error, g->Message);
 
-    msg[len]= '\0';
-    buf->copy(msg, (uint)strlen(msg), system_charset_info);
-  } else
-    buf->copy("Cannot retrieve msg", 19, system_charset_info);
+		buf->append(g->Message);
+	} else
+    buf->append("Cannot retrieve error message");
 
   DBUG_RETURN(false);
 } // end of get_error_message
diff --git a/storage/connect/javaconn.cpp b/storage/connect/javaconn.cpp
index d1be0ca1848..37a96351a3b 100644
--- a/storage/connect/javaconn.cpp
+++ b/storage/connect/javaconn.cpp
@@ -138,6 +138,16 @@ JAVAConn::JAVAConn(PGLOBAL g, PCSZ wrapper)
 //  EndCom();
 
 //  } // end of ~JAVAConn
+char *JAVAConn::GetUTFString(jstring s)
+{
+	char *str;
+	const char *utf = env->GetStringUTFChars(s, nullptr);
+
+	str = PlugDup(m_G, utf);
+	env->ReleaseStringUTFChars(s, utf);
+	env->DeleteLocalRef(s);
+	return str;
+}	// end of GetUTFString
 
 /***********************************************************************/
 /*  Screen for errors.                                                 */
@@ -152,17 +162,15 @@ bool JAVAConn::Check(jint rc)
 			"toString", "()Ljava/lang/String;");
 
 		if (exc != nullptr && tid != nullptr) {
-			jstring s = (jstring)env->CallObjectMethod(exc, tid);
-			const char *utf = env->GetStringUTFChars(s, (jboolean)false);
-			env->DeleteLocalRef(s);
-			Msg = PlugDup(m_G, utf);
+			s = (jstring)env->CallObjectMethod(exc, tid);
+			Msg = GetUTFString(s);
 		} else
 			Msg = "Exception occured";
 
 		env->ExceptionClear();
 	} else if (rc < 0) {
 		s = (jstring)env->CallObjectMethod(job, errid);
-		Msg = (char*)env->GetStringUTFChars(s, (jboolean)false);
+		Msg = GetUTFString(s);
 	} else
 		Msg = NULL;
 
diff --git a/storage/connect/javaconn.h b/storage/connect/javaconn.h
index 54b7c4e92b7..73812f6ab3b 100644
--- a/storage/connect/javaconn.h
+++ b/storage/connect/javaconn.h
@@ -90,6 +90,7 @@ public:
 
 	// Java operations
 protected:
+	char *GetUTFString(jstring s);
 	bool gmID(PGLOBAL g, jmethodID& mid, const char *name, const char *sig);
 	bool Check(jint rc = 0);
 
diff --git a/storage/connect/jdbconn.cpp b/storage/connect/jdbconn.cpp
index 33414ca74c2..24da65cf6a8 100644
--- a/storage/connect/jdbconn.cpp
+++ b/storage/connect/jdbconn.cpp
@@ -707,20 +707,13 @@ bool JDBConn::SetUUID(PGLOBAL g, PTDBJDBC tjp)
 			goto err;
 		}	// endif rc
 
-		// Returns 666 is case of error
-		//jtyp = env->CallIntMethod(job, typid, 5, nullptr);
-
-		//if (Check((jtyp == 666) ? -1 : 1)) {
-		//	sprintf(g->Message, "Getting jtyp: %s", Msg);
-		//	goto err;
-		//} // endif ctyp
-
+		// Should return 666 is case of error	(not done yet)
 		ctyp = (int)env->CallIntMethod(job, intfldid, 5, nullptr);
 
-		if (Check(ctyp)) {
-			sprintf(g->Message, "Getting ctyp: %s", Msg);
-			goto err;
-		} // endif ctyp
+		//if (Check((ctyp == 666) ? -1 : 1)) {
+		//	sprintf(g->Message, "Getting ctyp: %s", Msg);
+		//	goto err;
+		//} // endif ctyp
 
 		if (ctyp == 1111)
 			((PJDBCCOL)colp)->uuid = true;
@@ -836,11 +829,11 @@ bool JDBConn::Connect(PJPARM sop)
 		jstring s = (jstring)env->CallObjectMethod(job, qcid);
 
 		if (s != nullptr) {
-			char *qch = (char*)env->GetStringUTFChars(s, (jboolean)false);
+			char *qch = GetUTFString(s);
 			m_IDQuoteChar[0] = *qch;
 		} else {
 			s = (jstring)env->CallObjectMethod(job, errid);
-			Msg = (char*)env->GetStringUTFChars(s, (jboolean)false);
+			Msg = GetUTFString(s);
 		}	// endif s
 
 	}	// endif qcid
@@ -1018,7 +1011,7 @@ void JDBConn::SetColumnValue(int rank, PSZ name, PVAL val)
 			cn = nullptr;
 
 		if (cn) {
-			field = env->GetStringUTFChars(cn, (jboolean)false);
+			field = GetUTFString(cn);
 			val->SetValue_psz((PSZ)field);
 		} else
 			val->Reset();
@@ -1092,8 +1085,7 @@ void JDBConn::SetColumnValue(int rank, PSZ name, PVAL val)
 			cn = nullptr;
 
 		if (cn) {
-			const char *field = env->GetStringUTFChars(cn, (jboolean)false);
-			val->SetValue_psz((PSZ)field);
+			val->SetValue_psz((PSZ)GetUTFString(cn));
 		} else
 			val->Reset();
 
@@ -1372,19 +1364,19 @@ bool JDBConn::SetParam(JDBCCOL *colp)
 		for (i = 0, n = 0; i < size; i++) {
 			crp = qrp->Colresp;
 			js = (jstring)env->GetObjectArrayElement(s, n++);
-			sval = (PSZ)env->GetStringUTFChars(js, 0);
+			sval = GetUTFString(js);
 			crp->Kdata->SetValue(sval, i);
 			crp = crp->Next;
 			js = (jstring)env->GetObjectArrayElement(s, n++);
-			sval = (PSZ)env->GetStringUTFChars(js, 0);
+			sval = GetUTFString(js);
 			crp->Kdata->SetValue(sval, i);
 			crp = crp->Next;
 			js = (jstring)env->GetObjectArrayElement(s, n++);
-			sval = (PSZ)env->GetStringUTFChars(js, 0);
+			sval = GetUTFString(js);
 			crp->Kdata->SetValue(sval, i);
 			crp = crp->Next;
 			js = (jstring)env->GetObjectArrayElement(s, n++);
-			sval = (PSZ)env->GetStringUTFChars(js, 0);
+			sval = GetUTFString(js);
 			crp->Kdata->SetValue(sval, i);
 		}	// endfor i
 
@@ -1470,7 +1462,7 @@ bool JDBConn::SetParam(JDBCCOL *colp)
 				return NULL;
 			} // endif label
 
-			name = env->GetStringUTFChars(label, (jboolean)false);
+			name = GetUTFString(label);
 			crp = qrp->Colresp;                    // Column_Name
 			crp->Kdata->SetValue((char*)name, i);
 			n = env->GetIntArrayElements(val, 0);
diff --git a/storage/connect/jmgoconn.cpp b/storage/connect/jmgoconn.cpp
index 1731ccbeb8c..bd1ddadd80d 100644
--- a/storage/connect/jmgoconn.cpp
+++ b/storage/connect/jmgoconn.cpp
@@ -522,7 +522,7 @@ PSZ JMgoConn::GetDocument(void)
 		jdc = (jstring)env->CallObjectMethod(job, getdocid);
 
 		if (jdc)
-			doc = (PSZ)env->GetStringUTFChars(jdc, (jboolean)false);
+			doc = (PSZ)GetUTFString(jdc);
 
 	} // endif getdocid
 
@@ -690,7 +690,7 @@ jobject JMgoConn::MakeDoc(PGLOBAL g, PJNCOL jcp)
 
 		} // endif Jncolp
 
-		return parent;
+	return parent;
 } // end of MakeDoc
 
 /***********************************************************************/
@@ -807,7 +807,7 @@ PSZ JMgoConn::GetColumnValue(PSZ path)
 		fn = (jstring)env->CallObjectMethod(job, objfldid, jn);
 
 		if (fn)
-			fld = (PSZ)env->GetStringUTFChars(fn, (jboolean)false);
+			fld = (PSZ)GetUTFString(fn);
 
 	}	// endif objfldid
 
diff --git a/storage/connect/jsonudf.cpp b/storage/connect/jsonudf.cpp
index 6a3004f5c7a..26455d572b6 100644
--- a/storage/connect/jsonudf.cpp
+++ b/storage/connect/jsonudf.cpp
@@ -4631,7 +4631,7 @@ char *jbin_array(UDF_INIT *initid, UDF_ARGS *args, char *result,
 			bsp = NULL;
 
 		if (!bsp && (bsp = JbinAlloc(g, args, initid->max_length, NULL)))
-			strncpy(bsp->Msg, g->Message, 139);
+			strncpy(bsp->Msg, g->Message, BMX);
 
 		// Keep result of constant function
 		g->Xchk = (initid->const_item) ? bsp : NULL;
diff --git a/storage/connect/mysql-test/connect/r/jdbc.result b/storage/connect/mysql-test/connect/r/jdbc.result
index 895b4070d70..a16f2791e8f 100644
--- a/storage/connect/mysql-test/connect/r/jdbc.result
+++ b/storage/connect/mysql-test/connect/r/jdbc.result
@@ -238,8 +238,7 @@ DROP TABLE t1, connect.emp;
 CREATE TABLE t2 (command varchar(128) not null,number int(5) not null flag=1,message varchar(255) flag=2) ENGINE=CONNECT TABLE_TYPE=JDBC CONNECTION='jdbc:mariadb://localhost:PORT/connect' OPTION_LIST='User=root,Execsrc=1';
 SELECT * FROM t2 WHERE command='drop table tx1';
 command	number	message
-drop table tx1	0	Execute: java.sql.SQLSyntaxErrorException: Unknown table 'connect.tx1'
-Query is : drop table tx1
+drop table tx1	0	Execute: java.sql.SQLSyntaxErrorException: (conn:24) Unknown table 'connect.tx1'
 SELECT * FROM t2 WHERE command = 'create table tx1 (a int not null, b char(32), c double(8,2))';
 command	number	message
 create table tx1 (a int not null, b char(32), c double(8,2))	0	Affected rows
diff --git a/storage/connect/tabjmg.cpp b/storage/connect/tabjmg.cpp
index ba3e1c3e7c0..850d9e5fa9b 100644
--- a/storage/connect/tabjmg.cpp
+++ b/storage/connect/tabjmg.cpp
@@ -72,7 +72,7 @@ bool JMGDISC::Find(PGLOBAL g)
 bool JMGDISC::ColDesc(PGLOBAL g, jobject obj, char *pcn, char *pfmt,
 											int ncol, int k)
 {
-	const char *key;
+	const char *key, *utf;
 	char    colname[65];
 	char    fmt[129];
 	bool    rc = true;
@@ -101,7 +101,10 @@ bool JMGDISC::ColDesc(PGLOBAL g, jobject obj, char *pcn, char *pfmt,
 			continue;
 
 		jkey = (jstring)Jcp->env->CallObjectMethod(Jcp->job, bvnameid);
-		key = Jcp->env->GetStringUTFChars(jkey, (jboolean)false);
+		utf = Jcp->env->GetStringUTFChars(jkey, nullptr);
+		key = PlugDup(g, utf);
+		Jcp->env->ReleaseStringUTFChars(jkey, utf);
+		Jcp->env->DeleteLocalRef(jkey);
 
 		if (pcn) {
 			strncpy(colname, pcn, 64);
@@ -457,8 +460,8 @@ PSZ JMGCOL::GetJpath(PGLOBAL g, bool proj)
 				} else
 					*p2++ = *p1;
 
-				*p2 = 0;
-				return projpath;
+			*p2 = 0;
+			return projpath;
 		} else
 			return Jpath;
 
diff --git a/storage/connect/tabpivot.cpp b/storage/connect/tabpivot.cpp
index da5d134f347..9121a0453e5 100644
--- a/storage/connect/tabpivot.cpp
+++ b/storage/connect/tabpivot.cpp
@@ -107,12 +107,12 @@ bool PIVAID::SkipColumn(PCOLRES crp, char *skc)
 /***********************************************************************/
 PQRYRES PIVAID::MakePivotColumns(PGLOBAL g)
 {
-  char    *p, *query, *colname, *skc, buf[64];
-  int      ndif, nblin, w = 0;
-  bool     b = false;
-  PVAL     valp;   
-  PQRYRES  qrp;
-  PCOLRES *pcrp, crp, fncrp = NULL;
+	char    *p, *query, *colname, *skc, buf[64];
+	int      ndif, nblin, w = 0;
+	bool     b = false;
+	PVAL     valp;
+	PQRYRES  qrp;
+	PCOLRES *pcrp, crp, fncrp = NULL;
 
 	try {
 		// Are there columns to skip?
@@ -186,7 +186,7 @@ PQRYRES PIVAID::MakePivotColumns(PGLOBAL g)
 
 		} // endif picol
 
-	// Prepare the column list
+	  // Prepare the column list
 		for (pcrp = &Qryp->Colresp; crp = *pcrp; )
 			if (SkipColumn(crp, skc)) {
 				// Ignore this column
@@ -205,95 +205,95 @@ PQRYRES PIVAID::MakePivotColumns(PGLOBAL g)
 			} else
 				pcrp = &crp->Next;
 
-			if (!Rblkp) {
-				strcpy(g->Message, MSG(NO_DEF_PIVOTCOL));
-				goto err;
-			} else if (!fncrp) {
-				strcpy(g->Message, MSG(NO_DEF_FNCCOL));
-				goto err;
-			} // endif
+		if (!Rblkp) {
+			strcpy(g->Message, MSG(NO_DEF_PIVOTCOL));
+			goto err;
+		} else if (!fncrp) {
+			strcpy(g->Message, MSG(NO_DEF_FNCCOL));
+			goto err;
+		} // endif
 
-			if (Tabsrc) {
-				Myc.Close();
-				b = false;
+		if (Tabsrc) {
+			Myc.Close();
+			b = false;
 
-				// Before calling sort, initialize all
-				nblin = Qryp->Nblin;
+			// Before calling sort, initialize all
+			nblin = Qryp->Nblin;
 
-				Index.Size = nblin * sizeof(int);
-				Index.Sub = TRUE;                  // Should be small enough
+			Index.Size = nblin * sizeof(int);
+			Index.Sub = TRUE;                  // Should be small enough
 
-				if (!PlgDBalloc(g, NULL, Index))
-					goto err;
-
-				Offset.Size = (nblin + 1) * sizeof(int);
-				Offset.Sub = TRUE;                 // Should be small enough
-
-				if (!PlgDBalloc(g, NULL, Offset))
-					goto err;
-
-				ndif = Qsort(g, nblin);
-
-				if (ndif < 0)           // error
-					goto err;
-
-			} else {
-				// The query was limited, we must get pivot column values
-				// Returned values must be in their original character set
-		//  if (Myc.ExecSQL(g, "SET character_set_results=NULL", &w) == RC_FX)
-		//    goto err;
-
-				query = (char*)PlugSubAlloc(g, NULL, 0);
-				sprintf(query, "SELECT DISTINCT `%s` FROM `%s`", Picol, Tabname);
-				PlugSubAlloc(g, NULL, strlen(query) + 1);
-				Myc.FreeResult();
-
-				// Send the source command to MySQL
-				if (Myc.ExecSQL(g, query, &w) == RC_FX)
-					goto err;
-
-				// We must have a storage query to get pivot column values
-				if (!(qrp = Myc.GetResult(g, true)))
-					goto err;
-
-				Myc.Close();
-				b = false;
-
-				// Get the column list
-				crp = qrp->Colresp;
-				Rblkp = crp->Kdata;
-				ndif = qrp->Nblin;
-			} // endif Tabsrc
-
-			// Allocate the Value used to retieve column names
-			if (!(valp = AllocateValue(g, Rblkp->GetType(),
-				Rblkp->GetVlen(),
-				Rblkp->GetPrec())))
+			if (!PlgDBalloc(g, NULL, Index))
 				goto err;
 
-			// Now make the functional columns
-			for (int i = 0; i < ndif; i++) {
-				if (i) {
-					crp = (PCOLRES)PlugSubAlloc(g, NULL, sizeof(COLRES));
-					memcpy(crp, fncrp, sizeof(COLRES));
-				} else
-					crp = fncrp;
+			Offset.Size = (nblin + 1) * sizeof(int);
+			Offset.Sub = TRUE;                 // Should be small enough
 
-				// Get the value that will be the generated column name
-				if (Tabsrc)
-					valp->SetValue_pvblk(Rblkp, Pex[Pof[i]]);
-				else
-					valp->SetValue_pvblk(Rblkp, i);
+			if (!PlgDBalloc(g, NULL, Offset))
+				goto err;
 
-				colname = valp->GetCharString(buf);
-				crp->Name = PlugDup(g, colname);
-				crp->Flag = 1;
+			ndif = Qsort(g, nblin);
 
-				// Add this column
-				*pcrp = crp;
-				crp->Next = NULL;
-				pcrp = &crp->Next;
-			} // endfor i
+			if (ndif < 0)           // error
+				goto err;
+
+		} else {
+			// The query was limited, we must get pivot column values
+			// Returned values must be in their original character set
+	    //  if (Myc.ExecSQL(g, "SET character_set_results=NULL", &w) == RC_FX)
+	    //    goto err;
+
+			query = (char*)PlugSubAlloc(g, NULL, 0);
+			sprintf(query, "SELECT DISTINCT `%s` FROM `%s`", Picol, Tabname);
+			PlugSubAlloc(g, NULL, strlen(query) + 1);
+			Myc.FreeResult();
+
+			// Send the source command to MySQL
+			if (Myc.ExecSQL(g, query, &w) == RC_FX)
+				goto err;
+
+			// We must have a storage query to get pivot column values
+			if (!(qrp = Myc.GetResult(g, true)))
+				goto err;
+
+			Myc.Close();
+			b = false;
+
+			// Get the column list
+			crp = qrp->Colresp;
+			Rblkp = crp->Kdata;
+			ndif = qrp->Nblin;
+		} // endif Tabsrc
+
+		// Allocate the Value used to retieve column names
+		if (!(valp = AllocateValue(g, Rblkp->GetType(),
+				                          Rblkp->GetVlen(),
+				                          Rblkp->GetPrec())))
+			goto err;
+
+		// Now make the functional columns
+		for (int i = 0; i < ndif; i++) {
+			if (i) {
+				crp = (PCOLRES)PlugSubAlloc(g, NULL, sizeof(COLRES));
+				memcpy(crp, fncrp, sizeof(COLRES));
+			} else
+				crp = fncrp;
+
+			// Get the value that will be the generated column name
+			if (Tabsrc)
+				valp->SetValue_pvblk(Rblkp, Pex[Pof[i]]);
+			else
+				valp->SetValue_pvblk(Rblkp, i);
+
+			colname = valp->GetCharString(buf);
+			crp->Name = PlugDup(g, colname);
+			crp->Flag = 1;
+
+			// Add this column
+			*pcrp = crp;
+			crp->Next = NULL;
+			pcrp = &crp->Next;
+		} // endfor i
 
 		// We added ndif columns and removed 2 (picol and fncol)
 		Qryp->Nbcol += (ndif - 2);
@@ -306,10 +306,10 @@ PQRYRES PIVAID::MakePivotColumns(PGLOBAL g)
 	} // end catch
 
 err:
-  if (b)
-    Myc.Close();
+	if (b)
+		Myc.Close();
 
-  return NULL;
+	return NULL;
 } // end of MakePivotColumns
 
 /***********************************************************************/
diff --git a/storage/connect/tabutil.cpp b/storage/connect/tabutil.cpp
index 68b66aec31f..462a6fcd839 100644
--- a/storage/connect/tabutil.cpp
+++ b/storage/connect/tabutil.cpp
@@ -429,7 +429,7 @@ PTDB TDBPRX::GetSubTable(PGLOBAL g, PTABLE tabp, bool b)
       char buf[MAX_STR];
 
       strcpy(buf, g->Message);
-      sprintf(g->Message, "Error accessing %s.%s: %s", db, name, buf);
+      snprintf(g->Message, MAX_STR, "Error accessing %s.%s: %s", db, name, buf);
       hc->tshp = NULL;
       goto err;
       } // endif Define

From 4cbadaeaea5b47a128b2ddf8ca487dbfd978227d Mon Sep 17 00:00:00 2001
From: Alexander Barkov <bar@mariadb.com>
Date: Tue, 7 Aug 2018 11:55:51 +0400
Subject: [PATCH 119/164] MDEV-16891 EVENTs created with SQL_MODE=ORACLE fail
 to execute

---
 .../suite/compat/oracle/r/events.result       | 16 ++++++++++
 mysql-test/suite/compat/oracle/t/events.test  | 29 +++++++++++++++++++
 sql/event_data_objects.cc                     | 10 +++++--
 3 files changed, 52 insertions(+), 3 deletions(-)
 create mode 100644 mysql-test/suite/compat/oracle/r/events.result
 create mode 100644 mysql-test/suite/compat/oracle/t/events.test

diff --git a/mysql-test/suite/compat/oracle/r/events.result b/mysql-test/suite/compat/oracle/r/events.result
new file mode 100644
index 00000000000..7b4ad5679eb
--- /dev/null
+++ b/mysql-test/suite/compat/oracle/r/events.result
@@ -0,0 +1,16 @@
+set sql_mode='ORACLE';
+#
+# MDEV-16891 EVENTs created with SQL_MODE=ORACLE fail to execute
+#
+SET GLOBAL event_scheduler=off;
+SET sql_mode='ORACLE';
+CREATE TABLE t1 (a TIMESTAMP);
+CREATE EVENT e1
+ON SCHEDULE AT CURRENT_TIMESTAMP + INTERVAL 1 MICROSECOND
+DO INSERT INTO t1 VALUES(NOW());
+SET GLOBAL event_scheduler=on;
+SELECT COUNT(*) FROM t1;
+COUNT(*)
+1
+DROP TABLE t1;
+SET GLOBAL event_scheduler=off;
diff --git a/mysql-test/suite/compat/oracle/t/events.test b/mysql-test/suite/compat/oracle/t/events.test
new file mode 100644
index 00000000000..e898528636a
--- /dev/null
+++ b/mysql-test/suite/compat/oracle/t/events.test
@@ -0,0 +1,29 @@
+-- source include/not_embedded.inc
+
+set sql_mode='ORACLE';
+
+--echo #
+--echo # MDEV-16891 EVENTs created with SQL_MODE=ORACLE fail to execute
+--echo #
+
+SET GLOBAL event_scheduler=off;
+
+SET sql_mode='ORACLE';
+CREATE TABLE t1 (a TIMESTAMP);
+CREATE EVENT e1
+  ON SCHEDULE AT CURRENT_TIMESTAMP + INTERVAL 1 MICROSECOND
+  DO INSERT INTO t1 VALUES(NOW());
+SET GLOBAL event_scheduler=on;
+
+let $wait_timeout = 10;
+let $wait_condition =
+  SELECT COUNT(*) = 0
+  FROM INFORMATION_SCHEMA.EVENTS
+  WHERE event_schema = 'test' AND event_name = 'e1';
+--source include/wait_condition.inc
+
+SELECT COUNT(*) FROM t1;
+DROP TABLE t1;
+
+
+SET GLOBAL event_scheduler=off;
diff --git a/sql/event_data_objects.cc b/sql/event_data_objects.cc
index 28820f05aa3..86a710f87c6 100644
--- a/sql/event_data_objects.cc
+++ b/sql/event_data_objects.cc
@@ -1284,7 +1284,11 @@ Event_job_data::construct_sp_sql(THD *thd, String *sp_sql)
   */
   sp_sql->append(STRING_WITH_LEN("() SQL SECURITY INVOKER "));
 
+  if (thd->variables.sql_mode & MODE_ORACLE)
+    sp_sql->append(STRING_WITH_LEN(" AS BEGIN "));
   sp_sql->append(&body);
+  if (thd->variables.sql_mode & MODE_ORACLE)
+    sp_sql->append(STRING_WITH_LEN("; END"));
 
   DBUG_RETURN(thd->is_fatal_error);
 }
@@ -1387,9 +1391,6 @@ Event_job_data::execute(THD *thd, bool drop)
     goto end;
   }
 
-  if (construct_sp_sql(thd, &sp_sql))
-    goto end;
-
   /*
     Set up global thread attributes to reflect the properties of
     this Event. We can simply reset these instead of usual
@@ -1401,6 +1402,9 @@ Event_job_data::execute(THD *thd, bool drop)
   thd->variables.sql_mode= sql_mode;
   thd->variables.time_zone= time_zone;
 
+  if (construct_sp_sql(thd, &sp_sql))
+    goto end;
+
   thd->set_query(sp_sql.c_ptr_safe(), sp_sql.length());
 
   {

From 482d4da0a7a7f76e8ea9026978d49565f57a4192 Mon Sep 17 00:00:00 2001
From: Sachin <sachin.setiya@mariadb.com>
Date: Tue, 7 Aug 2018 16:05:48 +0530
Subject: [PATCH 120/164] MDEV-15127 AddressSanitizer: stack-buffer-overflow in
 base_list::push_back ..

Problem:-
 If we try to run this query with -WITH_ASAN=ON compiled server
  CREATE TABLE t1 (i INT);
  SET debug_dbug="+d,test_completely_invisible,test_invisible_index";
  CREATE TABLE t2 LIKE t1;

 This will generate a stack buffer overflow error.
  ==8922==ERROR: AddressSanitizer: stack-buffer-overflow on address #ADDR
Analyze:-
 Error is generated on this line
       if (((*last)=new list_node(info, &end_of_list)))
 So info is our Key*, &end_of_list is global variable and last == #ADDR
 So last is suspicious variable. And last is the variable present in alter_info
 ->key_list. Now the question is how this key_list->last gets wrong/
 different stack variable. In the backtrace,  we can see that key_list is
 generated in mysql_create_table_like_table by calling
 mysql_preapre_alter_table_function and dummy key_list is created by
 mysql_create_like_table. In the end on mysql_prepare_alter_table we call
   alter_info->key_list.swap(new_key_list);
 So there is two options either key_list is empty or not empty , IF it is not
 empty then there is no issues last ptr is replaced by thd->mem_root (allocated ptr)
 So problem arises when key_list is empty. It swaps the dummy last ptr by
 mysql_prepare_alter_table declared ptr. which is wrong.

Solution:-
 We wont swap variable if list does not have any element.
---
 mysql-test/main/invisible_field_debug.result | 5 +++++
 mysql-test/main/invisible_field_debug.test   | 6 ++++++
 sql/sql_list.h                               | 5 ++++-
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/mysql-test/main/invisible_field_debug.result b/mysql-test/main/invisible_field_debug.result
index 4c4ebffbdfc..b3c84d18333 100644
--- a/mysql-test/main/invisible_field_debug.result
+++ b/mysql-test/main/invisible_field_debug.result
@@ -369,3 +369,8 @@ t1	1	invisible	1	c	A	NULL	NULL	NULL	YES	BTREE
 t1	1	invisible_2	1	invisible	A	NULL	NULL	NULL	YES	BTREE		
 drop table t1;
 set @old_debug= @@debug_dbug;
+CREATE TABLE t1 (i INT );
+SET debug_dbug="+d,test_completely_invisible,test_invisible_index";
+CREATE TABLE t2 LIKE t1;
+SET debug_dbug= DEFAULT;
+DROP TABLE t1, t2;
diff --git a/mysql-test/main/invisible_field_debug.test b/mysql-test/main/invisible_field_debug.test
index 8674620e055..c1d6899d16e 100644
--- a/mysql-test/main/invisible_field_debug.test
+++ b/mysql-test/main/invisible_field_debug.test
@@ -270,3 +270,9 @@ explain select * from t1 where invisible =9;
 show indexes in t1;
 drop table t1;
 set @old_debug= @@debug_dbug;
+## MDEV 15127
+CREATE TABLE t1 (i INT );
+SET debug_dbug="+d,test_completely_invisible,test_invisible_index";
+CREATE TABLE t2 LIKE t1;
+SET debug_dbug= DEFAULT;
+DROP TABLE t1, t2;
diff --git a/sql/sql_list.h b/sql/sql_list.h
index 0219c226803..39a1c3375e0 100644
--- a/sql/sql_list.h
+++ b/sql/sql_list.h
@@ -278,10 +278,13 @@ public:
   */
   inline void swap(base_list &rhs)
   {
+    list_node **rhs_last=rhs.last;
     swap_variables(list_node *, first, rhs.first);
-    swap_variables(list_node **, last, rhs.last);
     swap_variables(uint, elements, rhs.elements);
+    rhs.last= last == &first ? &rhs.first : last;
+    last = rhs_last == &rhs.first ? &first : rhs_last;
   }
+
   inline list_node* last_node() { return *last; }
   inline list_node* first_node() { return first;}
   inline void *head() { return first->info; }

From eabf5230a258d4871f855c026b7bec02774d508b Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Tue, 7 Aug 2018 19:11:04 +0200
Subject: [PATCH 121/164] MDEV-16906 No groups to be reported (check your GNRs)
 - mysqld_multi does not see instances

make mysqld_multi to use same rules for my.cnf directories
that all other tools are using (see my_default.c).
---
 scripts/mysqld_multi.sh | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/scripts/mysqld_multi.sh b/scripts/mysqld_multi.sh
index 2ab70cdd4c3..10fcf2ae2b4 100644
--- a/scripts/mysqld_multi.sh
+++ b/scripts/mysqld_multi.sh
@@ -492,12 +492,19 @@ sub list_defaults_files
 
   return ($opt{file}) if exists $opt{file};
 
-  return      ('@sysconfdir@/my.cnf',
-               '@sysconfdir@/mysql/my.cnf',
-               '@prefix@/my.cnf',
-               ($ENV{MYSQL_HOME} ? "$ENV{MYSQL_HOME}/my.cnf" : undef),
-               $opt{'extra-file'},
-               ($ENV{HOME} ? "$ENV{HOME}/.my.cnf" : undef));
+  my @dirs;
+
+  # same rule as in mysys/my_default.c
+  if ('@sysconfdir@') {
+    push @dirs, '@sysconfdir@/my.cnf';
+  } else {
+    push @dirs, '/etc/my.cnf', '/etc/mysql/my.cnf';
+  }
+  push @dirs, "$ENV{MYSQL_HOME}/my.cnf" if $ENV{MYSQL_HOME};
+  push @dirs, $opt{'extra-file'} if $opt{'extra-file'};
+  push @dirs, "$ENV{HOME}/.my.cnf" if $ENV{HOME};
+
+  return @dirs;
 }
 
 

From 4b6dccc84ab3205e02d667bad63a74f70de04c9f Mon Sep 17 00:00:00 2001
From: Jacob Mathew <jacob.mathew@mariadb.com>
Date: Thu, 9 Aug 2018 00:04:09 -0700
Subject: [PATCH 122/164] MDEV-16398: Spider Creates Query With Non-Existent
 Function

The problem occurs because the statement generated by Spider used an
internal function name, ADD_TIME.

This problem has been corrected by the fix for bug MDEV-16878 within the
server, which enables Spider to generate the statement using the actual
SQL function name.  I have made some additional changes within Spider to fix
related problems that I observed while testing.

Author:
  Jacob Mathew.

First Reviewer:
  Alexander Barkov.

Second Reviewer:
  Kentoku Shiba.
---
 .../mysql-test/spider/include/init_spider.inc |  12 +--
 .../mysql-test/spider/r/auto_increment.result |   2 +-
 .../mysql-test/spider/r/timestamp.result      | 101 ++++++++++++++----
 .../spider/mysql-test/spider/t/timestamp.test |  85 +++++++++++++--
 .../mysql-test/spider/t/timestamp_init.inc    |  29 +++--
 storage/spider/spd_db_mysql.cc                |  18 +++-
 storage/spider/spd_table.cc                   |  23 ++--
 7 files changed, 213 insertions(+), 57 deletions(-)

diff --git a/storage/spider/mysql-test/spider/include/init_spider.inc b/storage/spider/mysql-test/spider/include/init_spider.inc
index f94bd51edd7..c4d171d418e 100644
--- a/storage/spider/mysql-test/spider/include/init_spider.inc
+++ b/storage/spider/mysql-test/spider/include/init_spider.inc
@@ -250,7 +250,7 @@ if (`SELECT IF($PLUGIN_VERSION = 3, 1, 0)`)
     format_id int not null default 0,
     gtrid_length int not null default 0,
     bqual_length int not null default 0,
-    data char(128) charset binary not null default '',
+    data binary(128) not null default '',
     status char(8) not null default '',
     PRIMARY KEY (data, format_id, gtrid_length),
     KEY idx1 (status)
@@ -260,7 +260,7 @@ if (`SELECT IF($PLUGIN_VERSION = 3, 1, 0)`)
     format_id int not null default 0,
     gtrid_length int not null default 0,
     bqual_length int not null default 0,
-    data char(128) charset binary not null default '',
+    data binary(128) not null default '',
     scheme char(64) not null default '',
     host char(64) not null default '',
     port char(5) not null default '',
@@ -282,7 +282,7 @@ if (`SELECT IF($PLUGIN_VERSION = 3, 1, 0)`)
     format_id int not null default 0,
     gtrid_length int not null default 0,
     bqual_length int not null default 0,
-    data char(128) charset binary not null default '',
+    data binary(128) not null default '',
     scheme char(64) not null default '',
     host char(64) not null default '',
     port char(5) not null default '',
@@ -337,8 +337,8 @@ if (`SELECT IF($PLUGIN_VERSION = 3, 1, 0)`)
   CREATE TABLE mysql.spider_link_mon_servers(
     db_name char(64) not null default '',
     table_name char(199) not null default '',
-    link_id char(5) not null default '',
-    sid int not null default 0,
+    link_id char(64) not null default '',
+    sid int unsigned not null default 0,
     server char(64) default null,
     scheme char(64) default null,
     host char(64) default null,
@@ -360,7 +360,7 @@ if (`SELECT IF($PLUGIN_VERSION = 3, 1, 0)`)
   CREATE TABLE mysql.spider_link_failed_log(
     db_name char(64) not null default '',
     table_name char(199) not null default '',
-    link_id int not null default 0,
+    link_id char(64) not null default '',
     failed_time timestamp not null default current_timestamp
   ) ENGINE=MYISAM DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
   DROP TABLE IF EXISTS mysql.spider_table_position_for_recovery;
diff --git a/storage/spider/mysql-test/spider/r/auto_increment.result b/storage/spider/mysql-test/spider/r/auto_increment.result
index cbc7fee0671..d968d58e657 100644
--- a/storage/spider/mysql-test/spider/r/auto_increment.result
+++ b/storage/spider/mysql-test/spider/r/auto_increment.result
@@ -154,7 +154,7 @@ col_a	col_b	col_c
 connection child2_1;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 argument
-select `col_a`,`col_b`,`col_c` from `auto_test_remote`.`tbl_a`
+select t0.`col_a` `col_a`,t0.`col_b` `col_b`,t0.`col_c` `col_c` from `auto_test_remote`.`tbl_a` t0
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
 SELECT col_a, col_b, col_c FROM tbl_a ORDER BY col_a;
 col_a	col_b	col_c
diff --git a/storage/spider/mysql-test/spider/r/timestamp.result b/storage/spider/mysql-test/spider/r/timestamp.result
index df40549cab2..bd1f442d462 100644
--- a/storage/spider/mysql-test/spider/r/timestamp.result
+++ b/storage/spider/mysql-test/spider/r/timestamp.result
@@ -40,8 +40,10 @@ SELECT 1;
 
 create table
 connection child2_1;
-CHILD2_1_DROP_TABLES
-CHILD2_1_CREATE_TABLES
+CHILD2_1_DROP_TABLE
+CHILD2_1_DROP_TABLE_F
+CHILD2_1_CREATE_TABLE
+CHILD2_1_CREATE_TABLE_F
 TRUNCATE TABLE mysql.general_log;
 connection master_1;
 DROP TABLE IF EXISTS tbl_a;
@@ -51,7 +53,12 @@ col_dt DATETIME,
 col_ts TIMESTAMP NOT NULL
 DEFAULT current_timestamp() ON UPDATE current_timestamp(),
 PRIMARY KEY(col_a),
+UNIQUE INDEX i_ts (col_ts)
 ) MASTER_1_ENGINE MASTER_1_AUTO_INCREMENT_2_1 MASTER_1_COMMENT_2_1
+CREATE TABLE tbl_f (
+col_d DATE,
+col_t TIME
+) ENGINE=Spider COMMENT='database "ts_test_remote", table "tbl_f", srv "s_2_1"'
 SHOW CREATE TABLE tbl_a;
 Table	Create Table
 tbl_a	CREATE TABLE `tbl_a` (
@@ -61,6 +68,12 @@ tbl_a	CREATE TABLE `tbl_a` (
   PRIMARY KEY (`col_a`),
   UNIQUE KEY `i_ts` (`col_ts`)
 ) ENGINE=SPIDER AUTO_INCREMENT=17 DEFAULT CHARSET=utf8mb4 COMMENT='database "ts_test_remote", table "tbl_a", srv "s_2_1"'
+SHOW CREATE TABLE tbl_f;
+Table	Create Table
+tbl_f	CREATE TABLE `tbl_f` (
+  `col_d` date DEFAULT NULL,
+  `col_t` time DEFAULT NULL
+) ENGINE=SPIDER DEFAULT CHARSET=latin1 COMMENT='database "ts_test_remote", table "tbl_f", srv "s_2_1"'
 
 Set a different time zone that has DST
 SET time_zone='+01:00';
@@ -101,7 +114,7 @@ col_a	col_dt	col_ts	unix_timestamp(col_ts)
 connection child2_1;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 argument
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a`
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
 SELECT col_a, col_dt, col_ts, unix_timestamp(col_ts) FROM tbl_a ORDER BY col_a;
 col_a	col_dt	col_ts	unix_timestamp(col_ts)
@@ -127,7 +140,7 @@ col_a	col_dt	col_ts	unix_timestamp(col_ts)
 connection child2_1;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 argument
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a`
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
 SELECT col_a, col_dt, col_ts, unix_timestamp(col_ts) FROM tbl_a ORDER BY col_a;
 col_a	col_dt	col_ts	unix_timestamp(col_ts)
@@ -153,7 +166,7 @@ col_a	col_dt	col_ts	unix_timestamp(col_ts)
 connection child2_1;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 argument
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a`
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
 SELECT col_a, col_dt, col_ts, unix_timestamp(col_ts) FROM tbl_a ORDER BY col_a;
 col_a	col_dt	col_ts	unix_timestamp(col_ts)
@@ -181,7 +194,7 @@ connection child2_1;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 argument
 select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` for update
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a`
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
 SELECT col_a, col_dt, col_ts, unix_timestamp(col_ts) FROM tbl_a ORDER BY col_a;
 col_a	col_dt	col_ts	unix_timestamp(col_ts)
@@ -239,13 +252,13 @@ col_a	col_dt	col_ts	unix_timestamp(col_ts)
 connection child2_1;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 argument
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` > '2017-12-31 23:00:00') order by `col_ts`
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` < '2018-10-28 01:30:00') order by `col_ts`
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where ('2018-10-28 01:30:00' > `col_ts`) order by `col_ts`
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where `col_ts` >= '2018-10-28 00:30:00' and `col_ts` <= '2018-10-28 01:30:00' and (`col_ts` between '2018-10-28 00:30:00'  and  '2018-10-28 01:30:00') order by `col_ts`
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where `col_ts` >= '2018-10-28 00:30:00' and `col_ts` <= '2018-10-28 01:30:00' and ((`col_ts` >= '2018-10-28 00:30:00') and (`col_ts` <= '2018-10-28 01:30:00')) order by `col_ts`
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` > '2018-03-25 01:00:00') order by `col_ts`
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` > '1970-01-01 00:00:01') order by `col_ts`
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0 where (t0.`col_ts` > '2017-12-31 23:00:00')
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0 where (t0.`col_ts` < '2018-10-28 01:30:00')
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0 where ('2018-10-28 01:30:00' > t0.`col_ts`)
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0 where (t0.`col_ts` between '2018-10-28 00:30:00'  and  '2018-10-28 01:30:00')
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0 where ((t0.`col_ts` >= '2018-10-28 00:30:00') and (t0.`col_ts` <= '2018-10-28 01:30:00'))
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0 where (t0.`col_ts` > '2018-03-25 01:00:00')
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0 where (t0.`col_ts` > '1970-01-01 00:00:01')
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
 SELECT col_a, col_dt, col_ts, unix_timestamp(col_ts) FROM tbl_a ORDER BY col_a;
 col_a	col_dt	col_ts	unix_timestamp(col_ts)
@@ -326,13 +339,13 @@ col_a	col_dt	col_ts	unix_timestamp(col_ts)
 connection child2_1;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 argument
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` > '2017-12-31 23:00:00')
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` < '2018-10-28 01:30:00')
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where ('2018-10-28 01:30:00' > `col_ts`)
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` between '2018-10-28 00:30:00'  and  '2018-10-28 01:30:00')
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where ((`col_ts` >= '2018-10-28 00:30:00') and (`col_ts` <= '2018-10-28 01:30:00'))
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` > '2018-03-25 01:00:00')
-select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` > '1970-01-01 00:00:01')
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0 where (t0.`col_ts` > '2017-12-31 23:00:00')
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0 where (t0.`col_ts` < '2018-10-28 01:30:00')
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0 where ('2018-10-28 01:30:00' > t0.`col_ts`)
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0 where (t0.`col_ts` between '2018-10-28 00:30:00'  and  '2018-10-28 01:30:00')
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0 where ((t0.`col_ts` >= '2018-10-28 00:30:00') and (t0.`col_ts` <= '2018-10-28 01:30:00'))
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0 where (t0.`col_ts` > '2018-03-25 01:00:00')
+select t0.`col_a` `col_a`,t0.`col_dt` `col_dt`,t0.`col_ts` `col_ts`,(unix_timestamp(t0.`col_ts`)) `unix_timestamp(col_ts)` from `ts_test_remote`.`tbl_a` t0 where (t0.`col_ts` > '1970-01-01 00:00:01')
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
 SELECT col_a, col_dt, col_ts, unix_timestamp(col_ts) FROM tbl_a ORDER BY col_a;
 col_a	col_dt	col_ts	unix_timestamp(col_ts)
@@ -343,6 +356,54 @@ col_a	col_dt	col_ts	unix_timestamp(col_ts)
 5	2018-10-28 02:30:00	2018-10-28 02:30:00	1540690200
 6	2038-01-19 04:14:07	2038-01-19 04:14:07	2147483647
 
+Test the TIMESTAMP function
+connection master_1;
+INSERT INTO tbl_f VALUES ('2018-06-24', '01:23:45'),
+('2018-06-24', '01:23:45'),
+('2018-08-01', '12:34:56');
+connection child2_1;
+TRUNCATE TABLE mysql.general_log;
+connection master_1;
+SELECT * FROM tbl_f;
+col_d	col_t
+2018-06-24	01:23:45
+2018-06-24	01:23:45
+2018-08-01	12:34:56
+SELECT TIMESTAMP(col_d, col_t) FROM tbl_f;
+TIMESTAMP(col_d, col_t)
+2018-06-24 01:23:45
+2018-06-24 01:23:45
+2018-08-01 12:34:56
+SELECT TIMESTAMP('2018-06-25', col_t) FROM tbl_f;
+TIMESTAMP('2018-06-25', col_t)
+2018-06-25 01:23:45
+2018-06-25 01:23:45
+2018-06-25 12:34:56
+SELECT TIMESTAMP(col_d, '10:43:21') FROM tbl_f;
+TIMESTAMP(col_d, '10:43:21')
+2018-06-24 10:43:21
+2018-06-24 10:43:21
+2018-08-01 10:43:21
+SELECT TIMESTAMP('2018-06-25', '10:43:21') FROM tbl_f;
+TIMESTAMP('2018-06-25', '10:43:21')
+2018-06-25 10:43:21
+2018-06-25 10:43:21
+2018-06-25 10:43:21
+connection child2_1;
+SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
+argument
+select t0.`col_d` `col_d`,t0.`col_t` `col_t` from `ts_test_remote`.`tbl_f` t0
+select (timestamp(t0.`col_d` , t0.`col_t`)) `TIMESTAMP(col_d, col_t)` from `ts_test_remote`.`tbl_f` t0
+select (timestamp('2018-06-25' , t0.`col_t`)) `TIMESTAMP('2018-06-25', col_t)` from `ts_test_remote`.`tbl_f` t0
+select (timestamp(t0.`col_d` , '10:43:21')) `TIMESTAMP(col_d, '10:43:21')` from `ts_test_remote`.`tbl_f` t0
+select (timestamp('2018-06-25' , '10:43:21')) `TIMESTAMP('2018-06-25', '10:43:21')` from `ts_test_remote`.`tbl_f` t0
+SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
+SELECT col_d, col_t FROM tbl_f;
+col_d	col_t
+2018-06-24	01:23:45
+2018-06-24	01:23:45
+2018-08-01	12:34:56
+
 Restore Time Zone settings
 connection master_1;
 SET GLOBAL time_zone=DEFAULT;
diff --git a/storage/spider/mysql-test/spider/t/timestamp.test b/storage/spider/mysql-test/spider/t/timestamp.test
index 7008638cc4a..225851a9d13 100644
--- a/storage/spider/mysql-test/spider/t/timestamp.test
+++ b/storage/spider/mysql-test/spider/t/timestamp.test
@@ -66,13 +66,17 @@ if ($USE_CHILD_GROUP2)
   if ($OUTPUT_CHILD_GROUP2)
   {
     --disable_query_log
-    echo CHILD2_1_DROP_TABLES;
-    echo CHILD2_1_CREATE_TABLES;
+    echo CHILD2_1_DROP_TABLE;
+    echo CHILD2_1_DROP_TABLE_F;
+    echo CHILD2_1_CREATE_TABLE;
+    echo CHILD2_1_CREATE_TABLE_F;
   }
   --disable_warnings
-  eval $CHILD2_1_DROP_TABLES;
+  eval $CHILD2_1_DROP_TABLE;
+  eval $CHILD2_1_DROP_TABLE_F;
   --enable_warnings
-  eval $CHILD2_1_CREATE_TABLES;
+  eval $CHILD2_1_CREATE_TABLE;
+  eval $CHILD2_1_CREATE_TABLE_F;
   if ($OUTPUT_CHILD_GROUP2)
   {
     --enable_query_log
@@ -98,7 +102,7 @@ echo CREATE TABLE tbl_a (
   col_ts TIMESTAMP NOT NULL
     DEFAULT current_timestamp() ON UPDATE current_timestamp(),
   PRIMARY KEY(col_a),
-  
+  UNIQUE INDEX i_ts (col_ts)
 ) MASTER_1_ENGINE MASTER_1_AUTO_INCREMENT_2_1 MASTER_1_COMMENT_2_1;
 eval CREATE TABLE tbl_a (
   col_a INT UNSIGNED NOT NULL AUTO_INCREMENT,
@@ -108,8 +112,17 @@ eval CREATE TABLE tbl_a (
   PRIMARY KEY(col_a),
   UNIQUE INDEX i_ts (col_ts)
 ) $MASTER_1_ENGINE $MASTER_1_AUTO_INCREMENT_2_1 $MASTER_1_COMMENT_2_1;
+echo CREATE TABLE tbl_f (
+  col_d DATE,
+  col_t TIME
+) $MASTER_1_ENGINE $MASTER_1_COMMENT_2_1_F;
+eval CREATE TABLE tbl_f (
+  col_d DATE,
+  col_t TIME
+) $MASTER_1_ENGINE $MASTER_1_COMMENT_2_1_F;
 --enable_query_log
 SHOW CREATE TABLE tbl_a;
+SHOW CREATE TABLE tbl_f;
 
 --echo
 --echo Set a different time zone that has DST
@@ -171,7 +184,7 @@ if ($USE_CHILD_GROUP2)
   {
     eval $CHILD2_1_SELECT_ARGUMENT1;
   }
-  eval $CHILD2_1_SELECT_TABLES;
+  eval $CHILD2_1_SELECT_TABLE;
   if (!$OUTPUT_CHILD_GROUP2)
   {
     --enable_query_log
@@ -214,7 +227,7 @@ if ($USE_CHILD_GROUP2)
   {
     eval $CHILD2_1_SELECT_ARGUMENT1;
   }
-  eval $CHILD2_1_SELECT_TABLES;
+  eval $CHILD2_1_SELECT_TABLE;
   if (!$OUTPUT_CHILD_GROUP2)
   {
     --enable_query_log
@@ -256,7 +269,7 @@ if ($USE_CHILD_GROUP2)
   {
     eval $CHILD2_1_SELECT_ARGUMENT1;
   }
-  eval $CHILD2_1_SELECT_TABLES;
+  eval $CHILD2_1_SELECT_TABLE;
   if (!$OUTPUT_CHILD_GROUP2)
   {
     --enable_query_log
@@ -299,7 +312,7 @@ if ($USE_CHILD_GROUP2)
   {
     eval $CHILD2_1_SELECT_ARGUMENT1;
   }
-  eval $CHILD2_1_SELECT_TABLES;
+  eval $CHILD2_1_SELECT_TABLE;
   if (!$OUTPUT_CHILD_GROUP2)
   {
     --enable_query_log
@@ -347,7 +360,7 @@ if ($USE_CHILD_GROUP2)
   {
     eval $CHILD2_1_SELECT_ARGUMENT1;
   }
-  eval $CHILD2_1_SELECT_TABLES;
+  eval $CHILD2_1_SELECT_TABLE;
   if (!$OUTPUT_CHILD_GROUP2)
   {
     --enable_query_log
@@ -425,7 +438,57 @@ if ($USE_CHILD_GROUP2)
   {
     eval $CHILD2_1_SELECT_ARGUMENT1;
   }
-  eval $CHILD2_1_SELECT_TABLES;
+  eval $CHILD2_1_SELECT_TABLE;
+  if (!$OUTPUT_CHILD_GROUP2)
+  {
+    --enable_query_log
+    --enable_result_log
+  }
+}
+
+--echo
+--echo Test the TIMESTAMP function
+--connection master_1
+INSERT INTO tbl_f VALUES ('2018-06-24', '01:23:45'),
+                         ('2018-06-24', '01:23:45'),
+                         ('2018-08-01', '12:34:56');
+if ($USE_CHILD_GROUP2)
+{
+  if (!$OUTPUT_CHILD_GROUP2)
+  {
+    --disable_query_log
+    --disable_result_log
+  }
+  --connection child2_1
+  if ($USE_GENERAL_LOG)
+  {
+    TRUNCATE TABLE mysql.general_log;
+  }
+  if (!$OUTPUT_CHILD_GROUP2)
+  {
+    --enable_query_log
+    --enable_result_log
+  }
+}
+--connection master_1
+SELECT * FROM tbl_f;
+SELECT TIMESTAMP(col_d, col_t) FROM tbl_f;
+SELECT TIMESTAMP('2018-06-25', col_t) FROM tbl_f;
+SELECT TIMESTAMP(col_d, '10:43:21') FROM tbl_f;
+SELECT TIMESTAMP('2018-06-25', '10:43:21') FROM tbl_f;
+if ($USE_CHILD_GROUP2)
+{
+  if (!$OUTPUT_CHILD_GROUP2)
+  {
+    --disable_query_log
+    --disable_result_log
+  }
+  --connection child2_1
+  if ($USE_GENERAL_LOG)
+  {
+    eval $CHILD2_1_SELECT_ARGUMENT1;
+  }
+  eval $CHILD2_1_SELECT_TABLE_F;
   if (!$OUTPUT_CHILD_GROUP2)
   {
     --enable_query_log
diff --git a/storage/spider/mysql-test/spider/t/timestamp_init.inc b/storage/spider/mysql-test/spider/t/timestamp_init.inc
index 12049ab34fa..811c34c9b9a 100644
--- a/storage/spider/mysql-test/spider/t/timestamp_init.inc
+++ b/storage/spider/mysql-test/spider/t/timestamp_init.inc
@@ -7,8 +7,11 @@
 --enable_warnings
 
 --let $MASTER_1_COMMENT_2_1_BACKUP= $MASTER_1_COMMENT_2_1
+--let $MASTER_1_COMMENT_2_1_F_BACKUP= $MASTER_1_COMMENT_2_1_F
 let $MASTER_1_COMMENT_2_1=
   COMMENT='database "ts_test_remote", table "tbl_a", srv "s_2_1"';
+let $MASTER_1_COMMENT_2_1_F=
+  COMMENT='database "ts_test_remote", table "tbl_f", srv "s_2_1"';
 let $MASTER_1_AUTO_INCREMENT_2_1=
   AUTO_INCREMENT=17 DEFAULT CHARSET=utf8mb4;
 let $MASTER_1_AUTO_INCREMENT1=
@@ -17,11 +20,15 @@ let $MASTER_1_AUTO_INCREMENT2=
   AUTO_INCREMENT=17 DEFAULT CHARSET=utf8mb4;
 let $CHILD2_1_AUTO_INCREMENT=
   AUTO_INCREMENT=17 DEFAULT CHARSET=utf8mb4;
---let $CHILD2_1_DROP_TABLES_BACKUP= $CHILD2_1_DROP_TABLES
-let $CHILD2_1_DROP_TABLES=
+--let $CHILD2_1_DROP_TABLE_BACKUP= $CHILD2_1_DROP_TABLE
+--let $CHILD2_1_DROP_TABLE_F_BACKUP= $CHILD2_1_DROP_TABLE_F
+let $CHILD2_1_DROP_TABLE=
   DROP TABLE IF EXISTS tbl_a;
---let $CHILD2_1_CREATE_TABLES_BACKUP= $CHILD2_1_CREATE_TABLES
-let $CHILD2_1_CREATE_TABLES=
+let $CHILD2_1_DROP_TABLE_F=
+  DROP TABLE IF EXISTS tbl_f;
+--let $CHILD2_1_CREATE_TABLE_BACKUP= $CHILD2_1_CREATE_TABLE
+--let $CHILD2_1_CREATE_TABLE_F_BACKUP= $CHILD2_1_CREATE_TABLE_F
+let $CHILD2_1_CREATE_TABLE=
   CREATE TABLE tbl_a (
     col_a INT UNSIGNED NOT NULL AUTO_INCREMENT,
     col_dt DATETIME,
@@ -30,15 +37,25 @@ let $CHILD2_1_CREATE_TABLES=
     PRIMARY KEY(col_a),
     UNIQUE INDEX i_ts (col_ts)
   ) $CHILD2_1_ENGINE $CHILD2_1_AUTO_INCREMENT;
---let $CHILD2_1_SELECT_TABLES_BACKUP= $CHILD2_1_SELECT_TABLES
-let $CHILD2_1_SELECT_TABLES=
+let $CHILD2_1_CREATE_TABLE_F=
+  CREATE TABLE tbl_f (
+    col_d DATE,
+    col_t TIME
+  ) $CHILD2_1_ENGINE;
+--let $CHILD2_1_SELECT_TABLE_BACKUP= $CHILD2_1_SELECT_TABLE
+--let $CHILD2_1_SELECT_TABLE_F_BACKUP= $CHILD2_1_SELECT_TABLE_F
+let $CHILD2_1_SELECT_TABLE=
   SELECT col_a, col_dt, col_ts, unix_timestamp(col_ts) FROM tbl_a ORDER BY col_a;
+let $CHILD2_1_SELECT_TABLE_F=
+  SELECT col_d, col_t FROM tbl_f;
 let $CHILD2_1_SELECT_ARGUMENT1=
   SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 let $CHILD2_1_DROP_INDEX=
   DROP INDEX i_ts ON tbl_a;
 let $CHILD2_1_SHOW_CREATE_TABLE=
   SHOW CREATE TABLE tbl_a;
+let $CHILD2_1_SHOW_CREATE_TABLE_F=
+  SHOW CREATE TABLE tbl_f;
 --let $OUTPUT_CHILD_GROUP2_BACKUP= $OUTPUT_CHILD_GROUP2
 --let $OUTPUT_CHILD_GROUP2= 1
 --let $USE_GENERAL_LOG_BACKUP= $USE_GENERAL_LOG
diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc
index db685c50b85..ada9aa20821 100644
--- a/storage/spider/spd_db_mysql.cc
+++ b/storage/spider/spd_db_mysql.cc
@@ -12877,7 +12877,8 @@ int spider_mysql_handler::append_list_item_select(
   uint dbton_id = spider_dbton_mysql.dbton_id, length;
   List_iterator_fast<Item> it(*select);
   Item *item;
-  Field **field_ptr;
+  Field *field;
+  const char *item_name;
   DBUG_ENTER("spider_mysql_handler::append_list_item_select");
   DBUG_PRINT("info",("spider this=%p", this));
   while ((item = it++))
@@ -12888,8 +12889,17 @@ int spider_mysql_handler::append_list_item_select(
     {
       DBUG_RETURN(error_num);
     }
-    field_ptr = fields->get_next_field_ptr();
-    length = (*field_ptr)->field_name.length;
+    field = *(fields->get_next_field_ptr());
+    if (field)
+    {
+      item_name = field->field_name.str;
+      length = field->field_name.length;
+    }
+    else
+    {
+      item_name = item->name.str;
+      length = item->name.length;
+    }
     if (str->reserve(
       SPIDER_SQL_COMMA_LEN + /* SPIDER_SQL_NAME_QUOTE_LEN */ 2 +
       SPIDER_SQL_SPACE_LEN + length
@@ -12897,7 +12907,7 @@ int spider_mysql_handler::append_list_item_select(
       DBUG_RETURN(HA_ERR_OUT_OF_MEM);
     str->q_append(SPIDER_SQL_SPACE_STR, SPIDER_SQL_SPACE_LEN);
     if ((error_num = spider_db_mysql_utility.append_name(str,
-      (*field_ptr)->field_name.str, length)))
+                                                         item_name, length)))
     {
       DBUG_RETURN(error_num);
     }
diff --git a/storage/spider/spd_table.cc b/storage/spider/spd_table.cc
index 283c9e7881d..1940dd5aad9 100644
--- a/storage/spider/spd_table.cc
+++ b/storage/spider/spd_table.cc
@@ -4679,15 +4679,20 @@ SPIDER_SHARE *spider_get_share(
       }
       if (!share->link_status_init)
       {
-        if (
-          (
-            table_share->tmp_table == NO_TMP_TABLE &&
-            sql_command != SQLCOM_DROP_TABLE &&
-            sql_command != SQLCOM_SHOW_CREATE
-          ) ||
-          /* for alter change link status */
-          sql_command == SQLCOM_ALTER_TABLE
-        ) {
+        /*
+          The link statuses need to be refreshed from the spider_tables table
+          if the operation:
+          - Is not a DROP TABLE on a permanent table; or
+          - Is an ALTER TABLE.
+
+          Note that SHOW CREATE TABLE is not excluded, because the commands
+          that follow it require up-to-date link statuses.
+        */
+        if ((table_share->tmp_table == NO_TMP_TABLE &&
+             sql_command != SQLCOM_DROP_TABLE) ||
+            /* for alter change link status */
+            sql_command == SQLCOM_ALTER_TABLE)
+        {
           SPD_INIT_ALLOC_ROOT(&mem_root, 4096, 0, MYF(MY_WME));
           init_mem_root = TRUE;
           if (

From 340c8a2a32dcbe0bb9bc88bd0a6bda5d5e76ed02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Thu, 9 Aug 2018 12:11:47 +0300
Subject: [PATCH 123/164] Remove set-but-not-used log_sys.log.state

---
 storage/innobase/include/log0log.h | 10 ----------
 storage/innobase/log/log0log.cc    |  1 -
 storage/innobase/log/log0recv.cc   |  5 -----
 3 files changed, 16 deletions(-)

diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h
index 046037f9241..fd8295cfe12 100644
--- a/storage/innobase/include/log0log.h
+++ b/storage/innobase/include/log0log.h
@@ -495,14 +495,6 @@ Stored in LOG_HEADER_FORMAT. */
 					header */
 #define LOG_FILE_HDR_SIZE	(4 * OS_FILE_LOG_BLOCK_SIZE)
 
-/** The state of a log group */
-enum log_group_state_t {
-	/** No corruption detected */
-	LOG_GROUP_OK,
-	/** Corrupted */
-	LOG_GROUP_CORRUPTED
-};
-
 typedef ib_mutex_t	LogSysMutex;
 typedef ib_mutex_t	FlushOrderMutex;
 
@@ -560,8 +552,6 @@ struct log_t{
     ulint				format;
     /** individual log file size in bytes, including the header */
     lsn_t				file_size;
-    /** corruption status */
-    log_group_state_t		state;
     /** lsn used to fix coordinates within the log group */
     lsn_t				lsn;
     /** the byte offset of the above lsn */
diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc
index b099e50cd9e..09ed2f16d7d 100644
--- a/storage/innobase/log/log0log.cc
+++ b/storage/innobase/log/log0log.cc
@@ -671,7 +671,6 @@ void log_t::files::create(ulint n_files)
     ? LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED
     : LOG_HEADER_FORMAT_CURRENT;
   file_size= srv_log_file_size;
-  state= LOG_GROUP_OK;
   lsn= LOG_START_LSN;
   lsn_offset= LOG_FILE_HDR_SIZE;
 
diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc
index 1f1829370c3..fdfdbfe3687 100644
--- a/storage/innobase/log/log0recv.cc
+++ b/storage/innobase/log/log0recv.cc
@@ -834,8 +834,6 @@ recv_find_max_checkpoint_0(ulint* max_field)
 			*max_field = field;
 			max_no = checkpoint_no;
 
-			log_sys.log.state = LOG_GROUP_OK;
-
 			log_sys.log.lsn = mach_read_from_8(
 				buf + LOG_CHECKPOINT_LSN);
 			log_sys.log.lsn_offset = static_cast<ib_uint64_t>(
@@ -930,8 +928,6 @@ recv_find_max_checkpoint(ulint* max_field)
 
 	buf = log_sys.checkpoint_buf;
 
-	log_sys.log.state = LOG_GROUP_CORRUPTED;
-
 	log_header_read(0);
 	/* Check the header page checksum. There was no
 	checksum in the first redo log format (version 0). */
@@ -998,7 +994,6 @@ recv_find_max_checkpoint(ulint* max_field)
 		if (checkpoint_no >= max_no) {
 			*max_field = field;
 			max_no = checkpoint_no;
-			log_sys.log.state = LOG_GROUP_OK;
 			log_sys.log.lsn = mach_read_from_8(
 				buf + LOG_CHECKPOINT_LSN);
 			log_sys.log.lsn_offset = mach_read_from_8(

From 925b6ee0485bdda7f15d0e9abb023cbb9e646763 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Wed, 8 Aug 2018 11:19:01 +0200
Subject: [PATCH 124/164] update C/C up to v3.0.6 tag

update the test to the new (correct) result
---
 libmariadb                                    | 2 +-
 mysql-test/r/session_tracker_last_gtid.result | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/libmariadb b/libmariadb
index 74fbbaefeb5..31eb49f1f4d 160000
--- a/libmariadb
+++ b/libmariadb
@@ -1 +1 @@
-Subproject commit 74fbbaefeb54a485786a7180e15c3995eeaeac0a
+Subproject commit 31eb49f1f4dff7841866f6eaaefe44f94e124d62
diff --git a/mysql-test/r/session_tracker_last_gtid.result b/mysql-test/r/session_tracker_last_gtid.result
index 795d0aaa2a2..deffeca1333 100644
--- a/mysql-test/r/session_tracker_last_gtid.result
+++ b/mysql-test/r/session_tracker_last_gtid.result
@@ -2,7 +2,13 @@
 # MDEV-15477: SESSION_SYSVARS_TRACKER does not track last_gtid
 #
 SET gtid_seq_no=1000;
+-- Tracker : SESSION_TRACK_SCHEMA
+-- test
+
 SET @@session.session_track_system_variables='last_gtid';
+-- Tracker : SESSION_TRACK_SCHEMA
+-- test
+
 create table t1 (a int) engine=innodb;
 -- Tracker : SESSION_TRACK_SYSTEM_VARIABLES
 -- last_gtid

From ba1c05cc0d5e7f2926fc55124eb54fc804e28f53 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Wed, 8 Aug 2018 17:29:35 +0200
Subject: [PATCH 125/164] compiler warning
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

warning: suggest a space before ‘;’ or explicit braces around empty body in ‘for’ statement
---
 extra/mariabackup/backup_copy.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc
index d32c1e6fde9..30e94153275 100644
--- a/extra/mariabackup/backup_copy.cc
+++ b/extra/mariabackup/backup_copy.cc
@@ -654,7 +654,7 @@ mkdirp(const char *pathname, int Flags, myf MyFlags)
 	parent[len-1]= 0;
 
 	for (p = parent + strlen(parent);
-	    !is_path_separator(*p) && p != parent; p--);
+	    !is_path_separator(*p) && p != parent; p--) ;
 
 	*p = 0;
 

From 3f4274f8cdbd23217065131c62b376ee1095bb5b Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Wed, 8 Aug 2018 17:39:45 +0200
Subject: [PATCH 126/164] SLES11 OpenSSL 0.9.8 support

---
 libmariadb    | 2 +-
 sql/mysqld.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/libmariadb b/libmariadb
index 31eb49f1f4d..d0f201451d0 160000
--- a/libmariadb
+++ b/libmariadb
@@ -1 +1 @@
-Subproject commit 31eb49f1f4dff7841866f6eaaefe44f94e124d62
+Subproject commit d0f201451d062b83912525b4821d82304b32ecb1
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index 8e2884e2048..2ad4e6f2a61 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -8195,7 +8195,7 @@ my_asn1_time_to_string(const ASN1_TIME *time, char *buf, size_t len)
   if (bio == NULL)
     return NULL;
 
-  if (!ASN1_TIME_print(bio, time))
+  if (!ASN1_TIME_print(bio, const_cast<ASN1_TIME*>(time)))
     goto end;
 
   n_read= BIO_read(bio, buf, (int) (len - 1));

From 10d347dc6a44635443157f23b12405e8b20c4b5e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= <otto@mariadb.org>
Date: Wed, 8 Aug 2018 12:06:01 +0300
Subject: [PATCH 127/164] Deb: Make libmariadb3 Breaks+Replaces
 libmariadbclient18 so upgrade pass

The package libmariadbclient18 contains the dialog.so plugin, which also
the new libmariadb3 ships. As they both use the exact same path the latter
must be marked as a with Breaks and Replaces relations ship.

Note: This fix is conservative hack for stable releases 10.2 and 10.3.
In 10.4, the development release at the time, we will clean up how the
libmariadb3 packaging and it's -compat packages are done to match that
what is done in downstream Debian official.
---
 debian/control | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/debian/control b/debian/control
index 2465cfd6b27..c23106b7694 100644
--- a/debian/control
+++ b/debian/control
@@ -105,6 +105,8 @@ Conflicts: libmariadbclient18 (<< 10.2.0),
            mariadb-server-5.2,
            mariadb-server-5.3,
            mariadb-server-5.5 (<< 5.5.33)
+Breaks: libmariadbclient18 (<< ${source:Version})
+Replaces: libmariadbclient18 (<< ${source:Version})
 Description: MariaDB database client library
  MariaDB is a fast, stable and true multi-user, multi-threaded SQL database
  server. SQL (Structured Query Language) is the most popular database query

From f30c5af42e4e72a3a0d8d5fb7b9fc884a249f292 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Thu, 9 Aug 2018 17:38:40 +0300
Subject: [PATCH 128/164] InnoDB: Correct an error message

Display the log record type in hexadecimal, not binary.
---
 storage/innobase/log/log0recv.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc
index c8976b193e2..7db7cf74d22 100644
--- a/storage/innobase/log/log0recv.cc
+++ b/storage/innobase/log/log0recv.cc
@@ -1531,7 +1531,8 @@ parse_log:
 		break;
 	default:
 		ptr = NULL;
-		ib::error() << "Incorrect log record type:" << type;
+		ib::error() << "Incorrect log record type "
+			<< ib::hex(unsigned(type));
 
 		recv_sys->found_corrupt_log = true;
 	}

From bdf50c3ebd287a8e126a4156fc893f57f8321b72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Thu, 9 Aug 2018 19:42:46 +0300
Subject: [PATCH 129/164] log_group_read_log_seg(): Validate the length

If the LOG_BLOCK_HDR_DATA_LEN field is corrupted, scanning the
log records could fail in strange ways. It is better to validate
the field as part of validating each log block.
---
 storage/innobase/log/log0recv.cc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc
index 7db7cf74d22..35a1dbcea80 100644
--- a/storage/innobase/log/log0recv.cc
+++ b/storage/innobase/log/log0recv.cc
@@ -723,6 +723,15 @@ loop:
 					  OS_FILE_LOG_BLOCK_SIZE, true);
 			}
 		}
+
+		ulint dl = log_block_get_data_len(buf);
+		if (dl < LOG_BLOCK_HDR_SIZE
+		    || (dl > OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_TRL_SIZE
+			&& dl != OS_FILE_LOG_BLOCK_SIZE)) {
+			recv_sys->found_corrupt_log = true;
+			end_lsn = *start_lsn;
+			break;
+		}
 	}
 
 	if (recv_sys->report(ut_time())) {

From 0e15ae1602d505af6a19aa859196384065311c74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 10 Aug 2018 10:00:47 +0300
Subject: [PATCH 130/164] recv_report_corrupt_log(): Avoid buffer overflow

If recv_sys_justify_left_parsing_buf() has been invoked, it is possible
that recv_previous_parsed_rec_offset is after the current offset.
In this case, we must not dump any bytes before the current record.
---
 storage/innobase/log/log0recv.cc | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc
index 35a1dbcea80..21fedd7eb5f 100644
--- a/storage/innobase/log/log0recv.cc
+++ b/storage/innobase/log/log0recv.cc
@@ -2308,30 +2308,30 @@ recv_report_corrupt_log(
 	ib::error() <<
 		"############### CORRUPT LOG RECORD FOUND ##################";
 
+	const ulint ptr_offset = ulint(ptr - recv_sys->buf);
+
 	ib::info() << "Log record type " << type << ", page " << space << ":"
 		<< page_no << ". Log parsing proceeded successfully up to "
 		<< recv_sys->recovered_lsn << ". Previous log record type "
 		<< recv_previous_parsed_rec_type << ", is multi "
 		<< recv_previous_parsed_rec_is_multi << " Recv offset "
-		<< (ptr - recv_sys->buf) << ", prev "
+		<< ptr_offset << ", prev "
 		<< recv_previous_parsed_rec_offset;
 
 	ut_ad(ptr <= recv_sys->buf + recv_sys->len);
 
 	const ulint	limit	= 100;
-	const ulint	before
-		= std::min(recv_previous_parsed_rec_offset, limit);
-	const ulint	after
-		= std::min(recv_sys->len - (ptr - recv_sys->buf), limit);
+	const ulint	prev_offset = std::min(recv_previous_parsed_rec_offset,
+					       ptr_offset);
+	const ulint	before = std::min(prev_offset, limit);
+	const ulint	after = std::min(recv_sys->len - ptr_offset, limit);
 
 	ib::info() << "Hex dump starting " << before << " bytes before and"
 		" ending " << after << " bytes after the corrupted record:";
 
-	ut_print_buf(stderr,
-		     recv_sys->buf
-		     + recv_previous_parsed_rec_offset - before,
-		     ptr - recv_sys->buf + before + after
-		     - recv_previous_parsed_rec_offset);
+	const byte* start = recv_sys->buf + prev_offset - before;
+
+	ut_print_buf(stderr, start, ulint(ptr - start) + after);
 	putc('\n', stderr);
 
 	if (!srv_force_recovery) {

From b853b4fd88b441f36eeb7eabdce79918ef10538a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 10 Aug 2018 13:02:01 +0300
Subject: [PATCH 131/164] Report InnoDB redo log corruption better

recv_parse_log_recs(): Check for corruption before checking for
end-of-log-buffer.

mlog_parse_initial_log_record(), page_cur_parse_delete_rec():
Flag corruption for out-of-bounds values, and let the caller
dump the corrupted redo log extract.
---
 storage/innobase/log/log0recv.cc  | 20 +++++++++-----------
 storage/innobase/mtr/mtr0log.cc   |  6 +++++-
 storage/innobase/page/page0cur.cc |  5 ++++-
 3 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc
index 21fedd7eb5f..f6569792721 100644
--- a/storage/innobase/log/log0recv.cc
+++ b/storage/innobase/log/log0recv.cc
@@ -2251,7 +2251,6 @@ recv_parse_log_rec(
 		*type, new_ptr, end_ptr, *space, *page_no, apply, NULL, NULL);
 
 	if (UNIV_UNLIKELY(new_ptr == NULL)) {
-
 		return(0);
 	}
 
@@ -2402,13 +2401,8 @@ loop:
 		len = recv_parse_log_rec(&type, ptr, end_ptr, &space,
 					 &page_no, apply, &body);
 
-		if (len == 0) {
-			return(false);
-		}
-
 		if (recv_sys->found_corrupt_log) {
-			recv_report_corrupt_log(
-				ptr, type, space, page_no);
+			recv_report_corrupt_log(ptr, type, space, page_no);
 			return(true);
 		}
 
@@ -2416,6 +2410,10 @@ loop:
 			return(true);
 		}
 
+		if (len == 0) {
+			return(false);
+		}
+
 		new_recovered_lsn = recv_calc_lsn_on_data_add(old_lsn, len);
 
 		if (new_recovered_lsn > recv_sys->scanned_lsn) {
@@ -2542,10 +2540,6 @@ loop:
 				&type, ptr, end_ptr, &space, &page_no,
 				false, &body);
 
-			if (len == 0) {
-				return(false);
-			}
-
 			if (recv_sys->found_corrupt_log
 			    || type == MLOG_CHECKPOINT
 			    || (*ptr & MLOG_SINGLE_REC_FLAG)) {
@@ -2559,6 +2553,10 @@ loop:
 				return(true);
 			}
 
+			if (len == 0) {
+				return(false);
+			}
+
 			recv_previous_parsed_rec_type = type;
 			recv_previous_parsed_rec_offset
 				= recv_sys->recovered_offset + total_len;
diff --git a/storage/innobase/mtr/mtr0log.cc b/storage/innobase/mtr/mtr0log.cc
index 8cfde15a3ba..9cc9b77d6a8 100644
--- a/storage/innobase/mtr/mtr0log.cc
+++ b/storage/innobase/mtr/mtr0log.cc
@@ -98,7 +98,11 @@ mlog_parse_initial_log_record(
 	}
 
 	*type = mlog_id_t(*ptr & ~MLOG_SINGLE_REC_FLAG);
-	ut_ad(*type <= MLOG_BIGGEST_TYPE || EXTRA_CHECK_MLOG_NUMBER(*type));
+	if (UNIV_UNLIKELY(*type > MLOG_BIGGEST_TYPE
+			  && !EXTRA_CHECK_MLOG_NUMBER(*type))) {
+		recv_sys->found_corrupt_log = true;
+		return NULL;
+	}
 
 	ptr++;
 
diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc
index b611d484306..3aea43bfba0 100644
--- a/storage/innobase/page/page0cur.cc
+++ b/storage/innobase/page/page0cur.cc
@@ -2250,7 +2250,10 @@ page_cur_parse_delete_rec(
 	offset = mach_read_from_2(ptr);
 	ptr += 2;
 
-	ut_a(offset <= UNIV_PAGE_SIZE);
+	if (UNIV_UNLIKELY(offset >= srv_page_size)) {
+		recv_sys->found_corrupt_log = true;
+		return NULL;
+	}
 
 	if (block) {
 		page_t*		page		= buf_block_get_frame(block);

From 29150e23912ef6e1fe386f53b778c9a5551c4790 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 10 Aug 2018 17:01:53 +0300
Subject: [PATCH 132/164] Revert part of
 b853b4fd88b441f36eeb7eabdce79918ef10538a

recv_parse_log_recs(): Do not check for corruption before
checking for end-of-log-buffer. For some reason, adding the
check to the logical-looking place would cause intermittent
recovery failures in the tests innodb.innodb-index and
innodb_gis.rtree_compress2.
---
 storage/innobase/log/log0recv.cc | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc
index f6569792721..45f0be93d4e 100644
--- a/storage/innobase/log/log0recv.cc
+++ b/storage/innobase/log/log0recv.cc
@@ -2401,6 +2401,10 @@ loop:
 		len = recv_parse_log_rec(&type, ptr, end_ptr, &space,
 					 &page_no, apply, &body);
 
+		if (len == 0) {
+			return(false);
+		}
+
 		if (recv_sys->found_corrupt_log) {
 			recv_report_corrupt_log(ptr, type, space, page_no);
 			return(true);
@@ -2410,10 +2414,6 @@ loop:
 			return(true);
 		}
 
-		if (len == 0) {
-			return(false);
-		}
-
 		new_recovered_lsn = recv_calc_lsn_on_data_add(old_lsn, len);
 
 		if (new_recovered_lsn > recv_sys->scanned_lsn) {
@@ -2540,6 +2540,10 @@ loop:
 				&type, ptr, end_ptr, &space, &page_no,
 				false, &body);
 
+			if (len == 0) {
+				return(false);
+			}
+
 			if (recv_sys->found_corrupt_log
 			    || type == MLOG_CHECKPOINT
 			    || (*ptr & MLOG_SINGLE_REC_FLAG)) {
@@ -2553,10 +2557,6 @@ loop:
 				return(true);
 			}
 
-			if (len == 0) {
-				return(false);
-			}
-
 			recv_previous_parsed_rec_type = type;
 			recv_previous_parsed_rec_offset
 				= recv_sys->recovered_offset + total_len;

From 9dd3e5ea3c7392562b75a40c7fb90b6750308b3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= <otto@mariadb.org>
Date: Mon, 6 Aug 2018 22:30:14 +0300
Subject: [PATCH 133/164] Deb: Make libmariadb3 Breaks+Replaces
 libmariadbclient18 so upgrade pass

The package libmariadbclient18 contains the dialog.so plugin, which also
the new libmariadb3 ships. As they both use the exact same path the latter
must be marked as a with Breaks and Replaces relations ship.

Note: This fix is conservative hack for stable releases 10.2 and 10.3.
In 10.4, the development release at the time, we will clean up how the
libmariadb3 packaging and it's -compat packages are done to match that
what is done in downstream Debian official.
---
 debian/control | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/debian/control b/debian/control
index e3d2dfcdb8f..173b8e55584 100644
--- a/debian/control
+++ b/debian/control
@@ -49,7 +49,9 @@ Conflicts: mariadb-galera-server-10.0 (<< 10.0.5),
            mariadb-server-5.3,
            mariadb-server-5.5 (<< 5.5.33),
            libmariadbclient18 (<< 10.2.0)
-Replaces: libmysqlclient19,
+Breaks: libmariadbclient18 (<< ${source:Version})
+Replaces: libmariadbclient18 (<< ${source:Version}),
+          libmysqlclient19,
           libmysqlclient20
 Provides: libmysqlclient19,
           libmysqlclient20

From 4cbf77e16ef9324acf85c01bc63023d62b821752 Mon Sep 17 00:00:00 2001
From: Elena Stepanova <elenst@montyprogram.com>
Date: Sun, 12 Aug 2018 04:56:06 +0300
Subject: [PATCH 134/164] Updated list of unstable tests for 10.2.17

---
 mysql-test/unstable-tests | 631 ++++++++++++++++++++++++--------------
 1 file changed, 399 insertions(+), 232 deletions(-)

diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests
index 00a914cc266..1f628ab240c 100644
--- a/mysql-test/unstable-tests
+++ b/mysql-test/unstable-tests
@@ -23,51 +23,77 @@
 #
 ##############################################################################
 
-# Based on 10.2 ee6ac4d313c5e8846c4f7b92a923154984528215
+# Based on bb-10.2-release f30c5af42e4e72a3a0d8d5fb7b9fc884a249f292
 
 main.alter_table                   : Modified in 10.2.16
-main.alter_table_errors            : Added in 10.2.15
+main.alter_table_trans             : MDEV-12084 - timeout
 main.analyze_stmt_slow_query_log   : MDEV-12237 - Wrong result
-main.assign_key_cache              : Added in 10.2.15
-main.assign_key_cache_debug        : Added in 10.2.15
+main.assign_key_cache              : Added in 10.0.36
+main.assign_key_cache_debug        : Added in 10.0.36
 main.auth_named_pipe               : MDEV-14724 - System error 2
 main.auto_increment                : Modified in 10.2.16
+main.bootstrap                     : Modified in 10.1.35, 10.2.17
 main.check                         : Modified in 10.2.16
-main.check_constraint              : Modified in 10.2.15
-main.connect                       : Modified in 10.2.15
-main.connect_debug                 : Added in 10.2.15
+main.check_constraint              : Modified in 10.2.17
+main.connect_debug                 : Added in 10.0.36
 main.connect2                      : MDEV-13885 - Server crash
+main.count_distinct2               : MDEV-11768 - timeout
+main.create_delayed                : MDEV-10605 - failed with timeout
 main.create_or_replace             : Modified in 10.2.16
-main.cte_nonrecursive              : Modified in 10.2.16
-main.cte_recursive                 : Modified in 10.2.16
-main.ctype_ucs                     : Modified in 10.2.15
-main.ctype_utf8mb4                 : Modified in 10.2.15
-main.derived_cond_pushdown         : Modified in 10.2.16
+main.create_replace_tmp            : Added in 10.2.17
+main.cte_nonrecursive              : Modified in 10.2.17
+main.cte_recursive                 : Modified in 10.2.17
+main.cte_recursive_not_embedded    : Added in 10.2.17
+main.ctype_binary                  : Modified in 10.1.35, 10.2.17
+main.ctype_eucjpms                 : Modified in 10.1.35, 10.2.17
+main.ctype_euckr                   : Modified in 10.1.35, 10.2.17
+main.ctype_gbk                     : Modified in 10.1.35, 10.2.17
+main.ctype_latin1                  : Modified in 10.1.35, 10.2.17
+main.ctype_ucs                     : Modified in 10.1.35, 10.2.17
+main.ctype_ujis                    : Modified in 10.1.35, 10.2.17
+main.ctype_upgrade                 : MDEV-16945 - Error upon mysql_upgrade
+main.ctype_utf16le                 : Modified in 10.1.35, 10.2.17
+main.ctype_utf16                   : MDEV-10675: timeout or extra warnings; modified in 10.1.35, 10.2.17
+main.ctype_utf32                   : Modified in 10.1.35, 10.2.17
+main.ctype_utf8                    : Modified in 10.1.35, 10.2.17
+main.ctype_utf8mb4                 : Modified in 10.1.35, 10.2.17
+main.debug_sync                    : MDEV-10607 - internal error
+main.derived                       : Modified in 10.1.35, 10.2.17
+main.derived_cond_pushdown         : Modified in 10.2.17
+main.derived_opt                   : MDEV-11768 - timeout
 main.distinct                      : MDEV-14194 - Crash; modified in 10.2.16
 main.drop_bad_db_type              : MDEV-15676 - Wrong result
 main.events_2                      : MDEV-13277 - Crash
 main.events_bugs                   : MDEV-12892 - Crash
+main.events_restart                : MDEV-12236 - Server shutdown problem
 main.events_slowlog                : MDEV-12821 - Wrong result
 main.explain_slowquerylog          : Modified in 10.2.16
-main.func_json                     : Modified in 10.2.16
-main.func_str                      : Modified in 10.2.15
-main.grant                         : Modified in 10.2.16
+main.func_json                     : Modified in 10.2.17
+main.gis                           : MDEV-13411 - wrong result on P8
+main.grant                         : Modified in 10.2.17
 main.grant2                        : Modified in 10.2.16
 main.grant_not_windows             : Added in 10.2.16
 main.having                        : Modified in 10.2.16
+main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown
+main.index_intersect_innodb        : MDEV-10643 - failed with timeout
 main.index_merge_innodb            : MDEV-7142 - Plan mismatch
 main.innodb_mysql_lock             : MDEV-7861 - Wrong result
 main.insert_select                 : Modified in 10.2.16
+main.join                          : Modified in 10.1.35, 10.2.17
+main.join_cache                    : Modified in 10.1.35, 10.2.17
+main.join_outer                    : Modified in 10.1.35, 10.2.17
 main.kill-2                        : MDEV-13257 - Wrong result
 main.kill_processlist-6619         : MDEV-10793 - Wrong result
 main.limit                         : Modified in 10.2.16
-main.lock                          : Modified in 10.2.15
 main.log_slow                      : MDEV-13263 - Wrong result
-main.mdev375                       : Modified in 10.2.15
+main.log_tables-big                : MDEV-13408 - wrong result
+main.max_statement_time            : Modified in 10.1.35, 10.2.17
 main.mdev-504                      : MDEV-15171 - warning
-main.myisam                        : Modified in 10.2.15
-main.myisam_recover                : Modified in 10.2.15
+main.mdev375                       : MDEV-10607 - sporadic "can't connect"
+main.merge                         : MDEV-10607 - sporadic "can't connect"
+main.myisam                        : Modified in 10.0.36
 main.mysql                         : Modified in 10.2.16
+main.mysql_client_test_comp        : MDEV-16641 - Error in exec
 main.mysql_client_test_nonblock    : CONC-208 - Error on Power; MDEV-15096 - exec failed
 main.mysql_cp932                   : Modified in 10.2.16
 main.mysql_upgrade_noengine        : MDEV-14355 - Wrong result
@@ -77,72 +103,76 @@ main.mysqld_option_err             : MDEV-12747 - Timeout
 main.mysqlhotcopy_myisam           : MDEV-10995 - Hang on debug
 main.mysqlslap                     : Modified in 10.2.16
 main.mysqltest                     : MDEV-13887 - Wrong result
+main.mysqltest_tracking_info       : Added in 10.2.17
 main.olap                          : Modified in 10.2.16
 main.openssl_1                     : MDEV-13492 - Unknown SSL error
 main.order_by_optimizer_innodb     : MDEV-10683 - Wrong result
-main.parser                        : Modified in 10.2.15
 main.partition_debug_sync          : MDEV-15669 - Deadlock found when trying to get lock
-main.partition_list                : Modified in 10.2.15
+main.partition_innodb_plugin       : MDEV-12901 - Valgrind warnings
 main.partition_open_files_limit    : Modified in 10.2.16
 main.ps                            : MDEV-11017 - Wrong result; modified in 10.2.16
 main.query_cache                   : MDEV-16180 - Wrong result
 main.query_cache_debug             : MDEV-15281 - Query cache is disabled
 main.range_vs_index_merge_innodb   : MDEV-15283 - Server has gone away
 main.read_only_innodb              : Modified in 10.2.16
-main.rename                        : Modified in 10.2.16
-main.select                        : MDEV-15430 - Wrong result with clang-4
-main.select_jcl6                   : MDEV-15430 - Wrong result with clang-4
-main.select_pkeycache              : MDEV-15430 - Wrong result with clang-4
+main.rename                        : Modified in 10.2.17
+main.reset_connection              : Added in 10.2.17
 main.selectivity                   : Modified in 10.2.16
+main.session_tracker_last_gtid     : Added in 10.2.17
 main.set_password                  : Added in 10.2.16
 main.set_statement                 : MDEV-13183 - Wrong result
 main.show_grants_with_plugin-7985  : Modified in 10.2.16
 main.shm                           : MDEV-12727 - Mismatch, ERROR 2013
 main.show_explain                  : MDEV-10674 - Wrong result code
 main.sp                            : MDEV-7866 - Mismatch; modified in 10.2.16
-main.sp-destruct                   : Modified in 10.2.15
-main.sp-innodb                     : Modified in 10.2.15
+main.sp-innodb                     : Modified in 10.0.36
+main.sp_notembedded                : MDEV-10607 - internal error
 main.sp-security                   : Modified in 10.2.16
 main.ssl_ca                        : MDEV-10895 - SSL connection error on Power
 main.ssl_cert_verify               : MDEV-13735 - Server crash
 main.ssl_connect                   : MDEV-13492 - Unknown SSL error
 main.ssl_timeout                   : MDEV-11244 - Crash
+main.stat_tables                   : Modified in 10.1.35, 10.2.17
 main.stat_tables_par               : MDEV-13266 - Wrong result
 main.stat_tables_par_innodb        : MDEV-14155 - Wrong rounding
-main.statistics                    : Modified in 10.2.15
+main.statistics                    : Modified in 10.1.35, 10.2.17
 main.statistics_close              : Modified in 10.2.16
 main.status                        : MDEV-13255 - Wrong result
-main.subselect4                    : Modified in 10.2.15
-main.subselect-crash_15755         : Added in 10.2.15
-main.subselect_innodb              : MDEV-10614 - Wrong result
-main.subselect_sj                  : Modified in 10.2.15
-main.subselect_sj2_mat             : Modified in 10.2.16
-main.symlink-myisam-11902          : MDEV-15098 - Error 40 from storage engine
+main.subselect                     : Modified in 10.1.35, 10.2.17
+main.subselect_innodb              : MDEV-10614 - Sporadic wrong results
+main.subselect_sj                  : Modified in 10.0.36
+main.subselect_sj_mat              : Modified in 10.1.35, 10.2.17
+main.subselect_sj2_mat             : Modified in 10.1.35, 10.2.17
+main.subselect4                    : Modified in 10.0.36
 main.tc_heuristic_recover          : MDEV-14189 - Wrong result
 main.trigger                       : Modified in 10.2.16
 main.type_bit                      : Modified in 10.2.16
 main.type_blob                     : MDEV-15195 - Wrong result; modified in 10.2.16
-main.type_datetime_hires           : MDEV-15430 - Wrong result with clang-4
+main.type_datetime                 : MDEV-14322 - wrong result
+main.type_datetime_hires           : MDEV-10687 - Timeout
 main.type_decimal                  : Modified in 10.2.16
-main.type_float                    : MDEV-15430 - Wrong result with clang-4
 main.type_int                      : Modified in 10.2.16
-main.type_time_hires               : MDEV-15430 - Wrong result with clang-4
-main.type_timestamp_hires          : MDEV-15430 - Wrong result with clang-4
+main.union                         : Modified in 10.1.35, 10.2.17
 main.userstat                      : MDEV-12904 - SSL errors
-main.variables                     : Modified in 10.2.15
-main.view                          : Modified in 10.2.15
-main.win                           : Modified in 10.2.15
+main.xa                            : MDEV-11769 - lock wait timeout
 
 #----------------------------------------------------------------
 
 archive.archive_bitfield           : MDEV-11771 - table is marked as crashed
+archive.archive_symlink            : MDEV-12170 - unexpected error on rmdir
+archive.discover                   : MDEV-10510 - Table is marked as crashed
 archive.mysqlhotcopy_archive       : MDEV-10995 - Hang on debug
 
+archive-test_sql_discovery.discover : MDEV-16817 - Table marked as crashed
+
 #----------------------------------------------------------------
 
 binlog.binlog_commit_wait          : MDEV-10150 - Mismatch
 binlog.binlog_flush_binlogs_delete_domain : MDEV-14431 - Wrong exit code
+binlog.binlog_killed                      : MDEV-12925 - Wrong result
+binlog.binlog_tmp_table_row               : Added in 10.1.35, 10.2.17
 binlog.binlog_xa_recover           : MDEV-8517 - Extra checkpoint
+binlog.load_data_stm_view          : MDEV-16948 - Wrong result
 
 #----------------------------------------------------------------
 
@@ -152,6 +182,9 @@ binlog_encryption.encrypted_master_switch_to_unencrypted : MDEV-14190 - Can't in
 binlog_encryption.encrypted_slave   : Modified in 10.2.16
 binlog_encryption.encryption_combo  : MDEV-14199 - Table is marked as crashed
 binlog_encryption.rpl_binlog_errors : MDEV-12742 - Crash
+binlog_encryption.rpl_checksum      : MDEV-16951 - Wrong result
+binlog_encryption.rpl_gtid_basic    : MDEV-16947 - Server failed to start
+binlog_encryption.rpl_loadfile      : MDEV-16645 - Timeout in include
 binlog_encryption.rpl_parallel      : MDEV-10653 - Timeout in include
 binlog_encryption.rpl_relayrotate   : MDEV-15194 - Timeout
 binlog_encryption.rpl_semi_sync     : MDEV-11673 - Valgrind
@@ -159,12 +192,13 @@ binlog_encryption.rpl_skip_replication : MDEV-13571 - Unexpected warning
 binlog_encryption.rpl_ssl           : MDEV-14507 - Timeouts
 binlog_encryption.rpl_stm_relay_ign_space : MDEV-13278 - Wrong result (test assertion)
 binlog_encryption.rpl_sync          : MDEV-13830 - Assertion failure
-binlog_encryption.rpl_typeconv      : Include file modified in 10.2.15
+binlog_encryption.rpl_typeconv      : MDEV-14362 - Lost connection to MySQL server during query
 
 #----------------------------------------------------------------
 
 connect.pivot                      : MDEV-14803 - Failed to discover table
 connect.vcol                       : MDEV-12374 - Fails on Windows
+connect.zip                        : MDEV-13884 - Wrong result
 
 #----------------------------------------------------------------
 
@@ -172,51 +206,81 @@ encryption.create_or_replace                     : MDEV-12694 - Timeout; MDEV-16
 encryption.debug_key_management                  : MDEV-13841 - Timeout
 encryption.encrypt_and_grep                      : MDEV-13765 - Wrong result
 encryption.innochecksum                          : MDEV-13644 - Assertion failure
-encryption.innodb-checksum-algorithm             : MDEV-12898 - Deadlock of threads
+encryption.innodb-checksum-algorithm             : MDEV-12898 - Deadlock of threads; MDEV-16896 - Server crash
 encryption.innodb-compressed-blob                : MDEV-14728 - Unable to get certificate
 encryption.innodb-discard-import                 : Modified in 10.2.16
 encryption.innodb_encrypt_log                    : MDEV-13725 - Wrong result
 encryption.innodb_encryption                     : MDEV-15675 - Timeout
 encryption.innodb-encryption-alter               : MDEV-13566 - Lock wait timeout
-encryption.innodb_encryption_discard_import      : MDEV-16116 - Wrong result; modified in 10.2.15
-encryption.innodb_encryption_filekeys            : MDEV-15673 - Timeout; modified in 10.2.15
+encryption.innodb_encryption_discard_import      : MDEV-16116 - Wrong result
 encryption.innodb_encryption-page-compression    : MDEV-12630 - crash or assertion failure
 encryption.innodb_encryption_row_compressed      : MDEV-16113 - Crash
+encryption.innodb_first_page                     : MDEV-10689 - Crash
 encryption.innodb-first-page-read                : MDEV-14356 - Timeout in wait condition
-encryption.innodb_lotoftables                    : Modified in 10.2.15
+encryption.innodb_lotoftables                    : MDEV-16111 - Wrong result
+encryption.innodb-page_encryption                : MDEV-10641 - mutex problem
 encryption.innodb-read-only                      : MDEV-16563 - Crash on startup
-encryption.innodb-redo-badkey                    : MDEV-13893 - Page cannot be decrypted; include file modified in 10.2.15
+encryption.innodb-redo-badkey                    : MDEV-13893 - Page cannot be decrypted
 encryption.innodb-redo-nokeys                    : Modified in 10.2.16
-encryption.innodb-remove-encryption              : MDEV-16493 - Timeout in wait condition; added in 10.2.15
+encryption.innodb-remove-encryption              : MDEV-16493 - Timeout in wait condition
+encryption.innodb_scrub                          : MDEV-8139 - scrubbing tests need fixing
+encryption.innodb_scrub_background               : MDEV-8139 - scrubbing tests need fixing
+encryption.innodb_scrub_compressed               : MDEV-8139 - scrubbing tests need fixing
 encryption.innodb-spatial-index                  : MDEV-13746 - Wrong result
 
 #----------------------------------------------------------------
 
+engines/iuds.*                     : Not maintained in timely manner
+engines/funcs.*                    : Not maintained in timely manner
 engines/rr_trx.*                   : MDEV-10998 - Not maintained
 
 #----------------------------------------------------------------
 
+federated.assisted_discovery       : Include file modified in 10.0.36
+federated.federated_bug_35333      : MDEV-13410 - Wrong result
 federated.federated_bug_585688     : MDEV-14805 - Server crash, MDEV-12907 - Valgrind
 federated.federated_innodb         : MDEV-10617 - Wrong checksum
+federated.federated_partition      : MDEV-10417 - Fails on Mips
 federated.federated_transactions   : MDEV-10617 - Wrong checksum
 federated.federatedx               : MDEV-10617 - Wrong checksum
 
 #----------------------------------------------------------------
 
+funcs_1.is_engines_federated         : Include file modified in 10.0.36
+funcs_1.memory_views                 : MDEV-11773 - timeout
 funcs_1.processlist_val_no_prot      : MDEV-11223 - Wrong result
+funcs_1.processlist_val_ps           : MDEV-12175 - Wrong plan
 
 funcs_2/charset.*                    : MDEV-10999 - Not maintained
 
+funcs_2.memory_charset               : MDEV-10290 - Timeout
+funcs_2.myisam_charset               : MDEV-11535 - Timeout
+
 #----------------------------------------------------------------
 
 galera.*                             : Suite is not stable yet
 galera_3nodes.*                      : Suite is not stable yet
 
+galera.galera_applier_ftwrl_table_alter : MDEV-13738 - MySQL server has gone away
+galera.galera_gcs_fc_limit              : MDEV-13877 - Timeout
+galera.galera_suspend_slave             : MDEV-13873 - Wrong error code
+galera.galera_toi_truncate              : MDEV-13743 - query 'reap' succeeded
+galera.galera_unicode_identifiers       : MDEV-13871 - Unknown database
+galera.galera_var_node_address          : MDEV-13880 - Failed to start mysqld
+galera.galera_wan                       : MDEV-13879 - Stray state UUID msg warnings
+galera.galera_wsrep_log_conficts        : MDEV-13874 - check-testcase failed
+galera.partition                        : MDEV-13881 - Wrong result
+galera.query_cache                      : MDEV-13883 - Wrong result
+galera.MW-328A                          : MDEV-13876 - Wrong result
+
 #----------------------------------------------------------------
 
+gcol.gcol_rollback                   : MDEV-16954 - Unknown storage engine 'InnoDB'
+gcol.gcol_update                     : Include file modified in 10.2.17
+gcol.innodb_virtual_basic            : MDEV-16950 - Failing assertion
 gcol.innodb_virtual_debug            : Modified in 10.2.16
-gcol.innodb_virtual_fk               : Modified in 10.2.15
-gcol.innodb_virtual_index            : Modified in 10.2.15
+gcol.innodb_virtual_debug_purge      : MDEV-16952 - Wrong result; modified in 10.2.17
+gcol.innodb_virtual_index            : Include file modified in 10.2.17
 
 #----------------------------------------------------------------
 
@@ -228,93 +292,101 @@ heap.heap_auto_increment             : Modified in 10.2.16
 
 #----------------------------------------------------------------
 
-innodb.101_compatibility             : MDEV-13891 - Wrong result
-innodb.alter_copy                    : MDEV-16181 - Assertion failure
-innodb.alter_foreign_crash           : Added in 10.2.16
-innodb.alter_kill                    : Added in 10.2.16
-innodb.alter_missing_tablespace      : Modified in 10.2.15
-innodb.alter_partitioned_debug       : Added in 10.2.15
-innodb.alter_partitioned_xa          : Added in 10.2.15
-innodb.alter_rename_files            : Added in 10.2.16
-innodb.analyze_table                 : Added in 10.2.16
-innodb.autoinc_persist               : MDEV-15282 - Assertion failure
-innodb.doublewrite                   : MDEV-12905 - Server crash; include file modified in 10.2.15
-innodb.foreign_key                   : Modified in 10.2.15
-innodb.group_commit_crash            : MDEV-14191 - InnoDB registration failed
+innodb.101_compatibility              : MDEV-13891 - Wrong result
+innodb.alter_copy                     : MDEV-16181 - Assertion failure
+innodb.alter_crash                    : MDEV-16944 - The process cannot access the file
+innodb.alter_foreign_crash            : Added in 10.2.16
+innodb.alter_kill                     : Added in 10.2.16
+innodb.alter_partitioned_xa           : Added in 10.0.36
+innodb.alter_rename_files             : Added in 10.2.16
+innodb.alter_kill                     : MDEV-16946 - Wrong result
+innodb.analyze_table                  : Added in 10.2.16
+innodb.autoinc_persist                : MDEV-15282 - Assertion failure
+innodb.binlog_consistent              : MDEV-10618 - Server fails to start
+innodb.doublewrite                    : MDEV-12905 - Server crash
+innodb.group_commit_crash             : MDEV-14191 - InnoDB registration failed
 innodb.group_commit_crash_no_optimize_thread : MDEV-13830	- Assertion failure
-innodb.innodb-64k-crash              : MDEV-13872 - Failure and crash on startup
-innodb.innodb-alter                  : Modified in 10.2.15
-innodb.innodb-alter-nullable         : Modified in 10.2.15
-innodb.innodb-alter-tempfile         : MDEV-15285 - Table already exists
-innodb.innodb-autoinc                : Modified in 10.2.16
-innodb.innodb-blob                   : Modified in 10.2.16
-innodb.innodb_bug14147491            : MDEV-11808 - Index is corrupt
-innodb.innodb_bug30423               : MDEV-7311 - Wrong result
-innodb.innodb_bug48024               : MDEV-14352 - Assertion failure
-innodb.innodb_bug54044               : Modified in 10.2.16
-innodb.innodb_bug59641               : MDEV-13830 - Assertion failure
+innodb.index_merge_threshold          : Include files modified in 10.2.17
+innodb.innodb-32k-crash               : MDEV-16953 - Corrupt log record found
+innodb.innodb-64k-crash               : MDEV-13872 - Failure and crash on startup
+innodb.innodb-alter                   : Modified in 10.0.36
+innodb.innodb-alter-debug             : MDEV-13182 - InnoDB: adjusting FSP_SPACE_FLAGS
+innodb.innodb-alter-table             : MDEV-10619 - Testcase timeout
+innodb.innodb-alter-tempfile          : MDEV-15285 - Table already exists
+innodb.innodb-autoinc                 : Modified in 10.2.16
+innodb.innodb-blob                    : Modified in 10.2.16
+innodb.innodb_bug14147491             : MDEV-11808 - Index is corrupt
+innodb.innodb_bug30423                : MDEV-7311 - Wrong result
+innodb.innodb_bug48024                : MDEV-14352 - Assertion failure
+innodb.innodb_bug54044                : Modified in 10.2.16
+innodb.innodb_bug59641                : MDEV-13830 - Assertion failure
 innodb.innodb_bulk_create_index_replication : MDEV-15273 - Slave failed to start
 innodb.innodb_defrag_stats_many_tables : MDEV-14198 - Table is full
-innodb.innodb_defragment_small       : Modified in 10.2.16
-innodb.innodb-get-fk                 : MDEV-13276 - Server crash
-innodb.innodb-index                  : Modified in 10.2.15
-innodb.innodb-index-online           : MDEV-14809 - Cannot save statistics
-innodb.innodb_information_schema     : MDEV-8851 - Wrong result
-innodb.innodb-isolation              : Modified in 10.2.15
-innodb.innodb_max_recordsize_32k     : MDEV-14801 - Operation failed
-innodb.innodb_max_recordsize_64k     : MDEV-15203 - Wrong result
-innodb.innodb-mdev7046               : Modified in 10.2.16
-innodb.innodb-online-alter-gis       : Modified in 10.2.16
+innodb.innodb_defragment_small        : Modified in 10.2.16
+innodb.innodb-fk                      : MDEV-13832 - Assertion failure on shutdown
+innodb.innodb-get-fk                  : MDEV-13276 - Server crash
+innodb.innodb-index-online            : MDEV-14809 - Cannot save statistics
+innodb.innodb_information_schema      : MDEV-8851 - Wrong result
+innodb.innodb_max_recordsize_32k      : MDEV-14801 - Operation failed
+innodb.innodb_max_recordsize_64k      : MDEV-15203 - Wrong result
+innodb.innodb-mdev7046                : Modified in 10.2.16
+innodb.innodb_monitor                 : MDEV-10939 - Testcase timeout
+innodb.innodb-online-alter-gis        : Modified in 10.2.16
 innodb.innodb-page_compression_default : MDEV-13644 - Assertion failure; modified in 10.2.16
-innodb.innodb-page_compression_lzma  : MDEV-14353 - Wrong result
+innodb.innodb-page_compression_lzma    : MDEV-14353 - Wrong result
 innodb.innodb-page_compression_snappy  : Modified in 10.2.16
-innodb.innodb_prefix_index_restart_server : Modified in 10.2.15
-innodb.innodb_query_cache            : Added in 10.2.16
-innodb.innodb_stats_persistent_debug : MDEV-14801 - Operation failed
-innodb.innodb-table-online           : MDEV-13894 - Wrong result
-innodb.innodb_sys_semaphore_waits    : MDEV-10331 - Semaphore wait
-innodb.innodb-wl5522-debug           : MDEV-14200 - Wrong errno
-innodb.innodb_zip_innochecksum2      : MDEV-13882 - Extra warnings
-innodb.innodb_zip_innochecksum3      : MDEV-14486 - Resource temporarily unavailable
-innodb.log_alter_table               : Include file modified in 10.2.15
-innodb.log_corruption                : MDEV-13251 - Wrong result
-innodb.log_data_file_size            : MDEV-14204 - Server failed to start; include file modified in 10.2.15
-innodb.log_file_name                 : MDEV-14193 - Exception; include file modified in 10.2.15
-innodb.log_file_name_debug           : Include file modified in 10.2.15
-innodb.log_file_size                 : MDEV-15668 - Not found pattern
-innodb.mdev-15707                    : Added in 10.2.15
-innodb.monitor                       : MDEV-16179 - Wrong result
-innodb.purge_secondary               : MDEV-15681 - Wrong result
-innodb.purge_thread_shutdown         : MDEV-13792 - Wrong result
-innodb.read_only_recovery            : MDEV-13886 - Server crash
-innodb.recovery_shutdown             : MDEV-15671 - Checksum mismatch in datafile
-innodb.rename_table                  : Modified in 10.2.16
-innodb.row_format_redundant          : MDEV-15192 - Trying to access missing tablespace
-innodb.stored_fk                     : Added in 10.2.15
-innodb.table_definition_cache_debug  : MDEV-14206 - Extra warning
-innodb.table_flags                   : MDEV-13572 - Wrong result
-innodb.temp_table_savepoint          : MDEV-16182 - Wrong result
-innodb.temporary_table               : MDEV-13265 - Wrong result; modified in 10.2.15
-innodb.undo_log                      : Modified in 10.2.15
-innodb.update_time                   : MDEV-14804 - Wrong result
-innodb.xa_recovery                   : MDEV-15279 - mysqld got exception
+innodb.innodb-page_compression_zip     : MDEV-10641 - mutex problem
+innodb.innodb_query_cache             : Added in 10.2.16
+innodb.innodb_stats                   : MDEV-10682 - wrong result
+innodb.innodb_stats_persistent        : Include file modified in 10.2.17
+innodb.innodb_stats_persistent_debug  : MDEV-14801 - Operation failed
+innodb.innodb-table-online            : MDEV-13894 - Wrong result
+innodb.innodb_sys_semaphore_waits     : MDEV-10331 - Semaphore wait
+innodb.innodb-wl5522                  : Modified in 10.1.35, 10.2.17
+innodb.innodb-wl5522-debug            : MDEV-14200 - Wrong errno
+innodb.innodb_zip_innochecksum2       : MDEV-13882 - Extra warnings
+innodb.innodb_zip_innochecksum3       : MDEV-14486 - Resource temporarily unavailable
+innodb.log_corruption                 : MDEV-13251 - Wrong result
+innodb.log_data_file_size             : MDEV-14204 - Server failed to start
+innodb.log_file_name                  : MDEV-14193 - Exception
+innodb.log_file_size                  : MDEV-15668 - Not found pattern
+innodb.monitor                        : MDEV-16179 - Wrong result
+innodb.purge_secondary                : MDEV-15681 - Wrong result; include file modified in 10.2.17
+innodb.purge_thread_shutdown          : MDEV-13792 - Wrong result
+innodb.read_only_recovery             : MDEV-13886 - Server crash
+innodb.recovery_shutdown              : MDEV-15671 - Checksum mismatch in datafile
+innodb.rename_table                   : Modified in 10.2.16
+innodb.row_format_redundant           : MDEV-15192 - Trying to access missing tablespace
+innodb.sp_temp_table                  : MDEV-16647 - Could not remove temporary table
+innodb.table_definition_cache_debug   : MDEV-14206 - Extra warning
+innodb.table_flags                    : MDEV-13572 - Wrong result
+innodb.temp_table_savepoint           : MDEV-16182 - Wrong result
+innodb.temporary_table                : MDEV-13265 - Wrong result; modified in 10.2.17
+innodb.truncate_purge_debug           : Include file modified in 10.2.17
+innodb.update_time                    : MDEV-14804 - Wrong result
+innodb.undo_log                       : Include file modified in 10.2.17
+innodb.xa_recovery                    : MDEV-15279 - mysqld got exception
 
-innodb_fts.basic                     : Added in 10.2.15
-innodb_fts.fulltext2                 : MDEV-14727 - Long semaphore wait
-innodb_fts.fulltext_misc             : MDEV-12636 - Valgrind
-innodb_fts.fulltext_table_evict      : Added in 10.2.15
-innodb_fts.innodb_fts_plugin         : MDEV-13888 - Errors in server log
+innodb_fts.basic                      : Added in 10.0.36
+innodb_fts.fts_kill_query             : Added in 10.1.35, 10.2.17
+innodb_fts.innodb-fts-fic             : MDEV-14154 - Assertion failure
+innodb_fts.innodb_fts_misc_debug      : MDEV-14156 - Unexpected warning
+innodb_fts.innodb_fts_plugin          : MDEV-13888 - Errors in server log
 innodb_fts.innodb_fts_stopword_charset : MDEV-13259 - Table crashed
-innodb_fts.sync                      : MDEV-14808 - Wrong result
+innodb_fts.sync                       : MDEV-14808 - Wrong result
+innodb_fts.sync_ddl                   : Added in 10.1.35, 10.2.17
 
+innodb_gis.create_spatial_index      : Modified in 10.2.17
+innodb_gis.kill_server               : MDEV-16941 - Checksum mismatch
+innodb_gis.rtree_compress            : Include file modified in 10.2.17
 innodb_gis.rtree_concurrent_srch     : MDEV-15284 - Wrong result with embedded
-innodb_gis.rtree_purge               : MDEV-15275 - Timeout
+innodb_gis.rtree_purge               : MDEV-15275 - Timeout; include file modified in 10.2.17
 innodb_gis.rtree_recovery            : MDEV-15274 - Error on check
 innodb_gis.rtree_split               : MDEV-14208 - Too many arguments
-innodb_gis.rtree_undo                : MDEV-14456 - Timeout in include file
-innodb_gis.types                     : MDEV-15679 - Table is marked as crashed; modified in 10.2.15
+innodb_gis.rtree_undo                : MDEV-14456 - Timeout in include file; include file modified in 10.2.17
+innodb_gis.types                     : MDEV-15679 - Table is marked as crashed
 
-innodb_zip.cmp_per_index             : MDEV-14490 - Table is marked as crashed; modified in 10.2.15
+innodb_zip.cmp_per_index             : MDEV-14490 - Table is marked as crashed
 innodb_zip.innochecksum_3            : MDEV-13279 - Extra warnings
 innodb_zip.wl6470_1                  : MDEV-14240 - Assertion failure
 innodb_zip.wl6501_1                  : MDEV-10891 - Can't create UNIX socket
@@ -332,51 +404,71 @@ maria.maria-autoinc                : Added in 10.2.16
 
 #----------------------------------------------------------------
 
-mariabackup.absolute_ibdata_paths   : Added in 10.2.15
 mariabackup.apply-log-only          : MDEV-14192 - Assertion failure
 mariabackup.apply-log-only-incr     : MDEV-14192 - Assertion failure; modified in 10.2.16
-mariabackup.backup_ssl              : MDEV-14192 - Assertion failure; added in 10.2.15
+mariabackup.backup_ssl              : MDEV-14192 - Assertion failure
 mariabackup.data_directory          : MDEV-15270 - Error on exec
 mariabackup.full_backup             : MDEV-16571 - Wrong result
 mariabackup.huge_lsn                : MDEV-15662 - Sequence number is in the future
 mariabackup.incremental_backup      : MDEV-14192 - Assertion failure
-mariabackup.incremental_encrypted   : MDEV-14188 - Wrong result, MDEV-15667 - timeout, MDEV-14192 - Assertion failure
+mariabackup.incremental_encrypted   : MDEV-15667 - Timeout; MDEV-14192 - Assertion failure; modified in 10.2.17
+mariabackup.innodb_log_optimize_ddl : Added in 10.2.17
 mariabackup.lock_ddl_per_table      : Modified in 10.2.16
-mariabackup.log_checksum_mismatch   : MDEV-16568 - Sequence number is in the future
 mariabackup.mdev-14447              : MDEV-15201 - Timeout; modified in 10.2.16
 mariabackup.partial_exclude         : MDEV-15270 - Error on exec
 mariabackup.partition_datadir       : Modified in 10.2.16
 mariabackup.rename_during_mdl_lock  : Added in 10.2.16
-mariabackup.unsupported_redo        : Modified in 10.2.15
+mariabackup.unsupported_redo        : Modified in 10.2.17
 mariabackup.xbstream                : MDEV-14192 - Crash
 mariabackup.xb_aws_key_management   : MDEV-15680 - Error: xtrabackup_copy_logfile() failed
 mariabackup.xb_page_compress        : MDEV-14810 - status: 1, errno: 11
+mariabackup.xb_partition            : MDEV-14192 - Assertion failure
 mariabackup.xb_compressed_encrypted : MDEV-14812 - Segmentation fault
-mariabackup.xb_file_key_management  : MDEV-16569 - Table doesn't exist
 mariabackup.xb_rocksdb              : Added in 10.2.16
 mariabackup.xb_rocksdb_datadir      : Added in 10.2.16
 mariabackup.xb_rocksdb_datadir_debug : Added in 10.2.16
 
 #----------------------------------------------------------------
 
-mroonga/storage.index_multiple_column_unique_datetime_index_read : MDEV-8643 - Valgrind
+mroonga/storage.column_datetime_32bit_2038                         : Wrong result on Alpha
+mroonga/storage.column_datetime_32bit_before_unix_epoch            : Wrong result on Alpha
+mroonga/storage.column_datetime_32bit_max                          : Wrong result on Alpha
+mroonga/storage.column_datetime_32bit_out_of_range                 : Wrong result on Alpha
+mroonga/storage.index_multiple_column_range_all_used_less_than                            : MDEV-16127 - Wrong result with GCC 8
+mroonga/storage.index_multiple_column_range_all_used_less_than_or_equal                   : MDEV-16127 - Wrong result with GCC 8
+mroonga/storage.index_multiple_column_range_partially_used_have_prefix_less_than          : MDEV-16127 - Wrong result with GCC 8
+mroonga/storage.index_multiple_column_range_partially_used_have_prefix_less_than_or_equal : MDEV-16127 - Wrong result with GCC 8
+mroonga/storage.index_multiple_column_range_partially_used_no_prefix_less_than            : MDEV-16127 - Wrong result with GCC 8
+mroonga/storage.index_multiple_column_range_partially_used_no_prefix_less_than_or_equal   : MDEV-16127 - Wrong result with GCC 8
+mroonga/storage.index_multiple_column_unique_date_32bit_equal      : Wrong result on Alpha
+mroonga/storage.index_multiple_column_unique_date_order_32bit_desc : Wrong result on Alpha
+mroonga/storage.index_multiple_column_unique_datetime_index_read   : MDEV-8643 - Valgrind
+mroonga/storage.optimization_order_limit_optimized_datetime_less_than                     : MDEV-16127 - Wrong result with GCC 8
+mroonga/storage.optimization_order_limit_optimized_datetime_less_than_or_equal            : MDEV-16127 - Wrong result with GCC 8
+mroonga/storage.repair_table_no_index_file                         : MDEV-9364 - Wrong result
+
+mroonga/wrapper.repair_table_no_index_file                         : MDEV-14807 - Wrong error message
 
 #----------------------------------------------------------------
 
 multi_source.gtid                  : MDEV-14202 - Crash
 multi_source.info_logs             : MDEV-12629 - Valgrind, MDEV-10042 - wrong result
+multi_source.multisource           : MDEV-10417 - Fails on Mips
 multi_source.reset_slave           : MDEV-10690 - Wrong result
 multi_source.simple                : MDEV-4633 - Wrong result
+multi_source.status_vars           : MDEV-4632 - failed while waiting for Slave_received_heartbeats
 
 #----------------------------------------------------------------
 
-parts.partition_alter_innodb         : Include file modified in 10.2.15
-parts.partition_alter_maria          : Include file modified in 10.2.15
-parts.partition_alter_myisam         : Include file modified in 10.2.15
+parts.alter_data_directory_innodb    : Added in 10.1.35, 10.2.17
+parts.partition_alter2_2_maria       : MDEV-14364 - Lost connection to MySQL server during query
 parts.partition_auto_increment_archive : MDEV-16491 - Marked as crashed and should be repaired
 parts.partition_auto_increment_maria : MDEV-14430 - Extra warning
 parts.partition_debug_innodb         : MDEV-10891 - Can't create UNIX socket; MDEV-15095 - Table doesn't exist
-parts.show_create                    : Added in 10.2.15
+parts.partition_exch_qa_10           : MDEV-11765 - wrong result
+parts.partition_innodb_status_file   : MDEV-12901 - Valgrind
+parts.partition_special_innodb       : MDEV-16942 - Timeout
+parts.truncate_locked                : Added in 10.1.35, 10.2.17
 
 #----------------------------------------------------------------
 
@@ -388,17 +480,20 @@ perfschema.bad_option_1                        : MDEV-13892 - Timeout
 perfschema.bad_option_3                        : MDEV-12728 - Timeout on Power
 perfschema.bad_option_5                        : MDEV-14197 - Timeout
 perfschema.dml_file_instances                  : MDEV-15179 - Wrong result
+perfschema.func_file_io                        : MDEV-5708 - fails for s390x
+perfschema.func_mutex                          : MDEV-5708 - fails for s390x
 perfschema.hostcache_ipv4_addrinfo_again_allow : MDEV-12759 - Crash
-perfschema.hostcache_ipv4_max_con              : Modified in 10.2.15
 perfschema.hostcache_ipv6_addrinfo_again_allow : MDEV-12752 - Crash
 perfschema.hostcache_ipv6_addrinfo_bad_allow   : MDEV-13260 - Crash
-perfschema.hostcache_ipv6_max_con              : Modified in 10.2.15
 perfschema.hostcache_ipv6_ssl                  : MDEV-10696 - Crash
 perfschema.partition                           : Added in 10.2.16
-perfschema.setup_actors                        : MDEV-10679 - Crash
+perfschema.privilege_table_io                  : MDEV-13184 - Extra lines
+perfschema.rpl_gtid_func                       : MDEV-16897 - Wrong result
 perfschema.socket_connect                      : MDEV-15677 - Wrong result
 perfschema.socket_summary_by_event_name_func   : MDEV-10622 - Wrong result
+perfschema.stage_mdl_global                    : MDEV-11803 - wrong result on slow builders
 perfschema.stage_mdl_procedure                 : MDEV-11545 - Missing row
+perfschema.stage_mdl_table                     : MDEV-12638 - Wrong result
 perfschema.threads_mysql                       : MDEV-10677 - Wrong result
 
 #----------------------------------------------------------------
@@ -408,182 +503,254 @@ perfschema_stress.*                : MDEV-10996 - Not maintained
 #----------------------------------------------------------------
 
 plugins.feedback_plugin_send       : MDEV-7932, MDEV-11118 - Connection problems and such
-plugins.processlist                : MDEV-16574 - Wrong result; modified in 10.2.16
-plugins.server_audit               : Modified in 10.2.15
+plugins.processlist                : MDEV-16574 - Wrong result; modified in 10.2.17
+plugins.server_audit               : MDEV-9562 - crashes on sol10-sparc
 plugins.thread_pool_server_audit   : MDEV-14295 - Wrong result
 
 #----------------------------------------------------------------
 
 rocksdb.2pc_group_commit             : MDEV-14455 - Wrong result
-rocksdb.allow_no_primary_key         : Modified in 10.2.16
+rocksdb.add_index_inplace            : MDEV-16648 - Server crash
+rocksdb.allow_no_primary_key         : MDEV-16634 - Server crash; modified in 10.2.16
+rocksdb.allow_no_primary_key_with_sk : MDEV-16639 - Server crash
 rocksdb.analyze_table                : Modified in 10.2.16
+rocksdb.autoinc_crash_safe_partition : MDEV-16639, MDEV-16637 - Server crash
 rocksdb.autoinc_debug                : MDEV-16203 - Wrong result
+rocksdb.autoinc_secondary            : MDEV-16638 - Server crash
 rocksdb.autoinc_vars_thread          : MDEV-16573 - Debug sync timed out
 rocksdb.bloomfilter2                 : MDEV-16564 - Wrong result
+rocksdb.bloomfilter4                 : MDEV-16649 - Server crash
 rocksdb.bulk_load_errors             : MDEV-16575 - Wrong result
 rocksdb.check_ignore_unknown_options : Modified in 10.2.16
 rocksdb.deadlock                     : MDEV-16033 - Timeout
 rocksdb.drop_index_inplace           : MDEV-14162 - Crash on shutdown
 rocksdb.drop_table                   : MDEV-14308 - Timeout
+rocksdb.drop_table2                  : MDEV-16631 - Server crash
+rocksdb.drop_table3                  : MDEV-16949 - Server crash
 rocksdb.issue255                     : MDEV-16577 - Wrong plan; modified in 10.2.16
+rocksdb.mariadb_ignore_dirs          : MDEV-16639 - Server crash
 rocksdb.mariadb_port_fixes           : MDEV-16387 - Wrong plan; modified in 10.2.16
+rocksdb.max_open_files               : MDEV-16639 - Server crash
 rocksdb.read_only_tx                 : MDEV-16565 - Server crash
+rocksdb.rocksdb_cf_options           : MDEV-16639 - Server crash
+rocksdb.rocksdb_cf_per_partition     : MDEV-16636 - Wrong result
 rocksdb.rocksdb_parts                : MDEV-13843 - Wrong result
+rocksdb.singledelete                 : MDEV-16633 - Server crash
 rocksdb.truncate_table3              : MDEV-14506 - Lost connection to server
 rocksdb.ttl_primary_read_filtering   : MDEV-16560 - Wrong result
+rocksdb.ttl_secondary                : MDEV-16943 - Timeout
 rocksdb.ttl_secondary_read_filtering : MDEV-16560 - Wrong result
 rocksdb.unique_check                 : MDEV-16576 - Wrong errno
+rocksdb.use_direct_reads_writes      : MDEV-16646 - Server crash
+
+rocksdb_rpl.mdev12179                : MDEV-16632 - Server crash
+rocksdb_rpl.rpl_binlog_xid_count     : MDEV-16644 - Server crash
 
 rocksdb_sys_vars.rocksdb_remove_mariabackup_checkpoint_basic  : Added in 10.2.16
+rocksdb_sys_vars.rocksdb_update_cf_options_basic : MDEV-16955 - Bytes lost
+rocksdb_sys_vars.rocksdb_update_cf_options       : MDEV-16955 - Bytes lost
 
 #----------------------------------------------------------------
 
 roles.acl_load_mutex-5170            : Modified in 10.2.16
+roles.create_and_grant_role          : MDEV-11772 - wrong result
 roles.grant_revoke_current           : Modified in 10.2.16
 roles.set_default_role_ps-6960       : Modified in 10.2.16
 
 #----------------------------------------------------------------
 
-rpl.rename                           : Added in 10.2.16
-rpl.rpl_binlog_errors                : MDEV-12742 - Crash
-rpl.rpl_binlog_index                 : MDEV-9501 - Failed registering on master
-rpl.rpl_colSize                      : MDEV-16112 - Server crash
-rpl.rpl_ctype_latin1                 : MDEV-14813 - Wrong result on Mac
-rpl.rpl_do_grant                     : Modified in 10.2.16
-rpl.rpl_domain_id_filter_io_crash    : MDEV-12729 - Timeout in include file, MDEV-13677 - Server crash
-rpl.rpl_domain_id_filter_restart     : MDEV-10684 - Wrong result
-rpl.rpl_extra_col_master_innodb      : MDEV-16570 - Extra warning
-rpl.rpl_extra_col_master_myisam      : MDEV-14203 - Extra warning
-rpl.rpl_gtid_crash                   : MDEV-9501 - Failed registering on master, MDEV-13643 - Lost connection
-rpl.rpl_gtid_delete_domain           : MDEV-14463 - Timeout
-rpl.rpl_gtid_errorhandling           : MDEV-13261 - Crash
-rpl.rpl_gtid_reconnect               : MDEV-14497 - Crash
-rpl.rpl_insert_id                    : MDEV-15197 - Wrong result
-rpl.rpl_insert_id_pk                 : MDEV-16567 - Assertion failure
-rpl.rpl_mariadb_slave_capability     : MDEV-11018 - Extra lines in binlog
-rpl.rpl_mdev6020                     : MDEV-15272 - Server crash
-rpl.rpl_mixed_mixing_engines         : MDEV-14489 - Sync slave with master failed
+rpl.last_insert_id                    : MDEV-10625 - warnings in error log
+rpl.rename                            : Added in 10.2.16
+rpl.rpl_15867                         : Added in 10.2.17
+rpl.rpl_auto_increment                : MDEV-10417 - Fails on Mips
+rpl.rpl_auto_increment_bug45679       : MDEV-10417 - Fails on Mips
+rpl.rpl_auto_increment_update_failure : MDEV-10625 - warnings in error log
+rpl.rpl_binlog_errors                 : MDEV-12742 - Crash
+rpl.rpl_binlog_index                  : MDEV-9501 - Failed registering on master
+rpl.rpl_colSize                       : MDEV-16112 - Server crash
+rpl.rpl_ctype_latin1                  : MDEV-14813 - Wrong result on Mac
+rpl.rpl_ddl                           : MDEV-10417 - Fails on Mips
+rpl.rpl_do_grant                      : Modified in 10.2.16
+rpl.rpl_domain_id_filter_io_crash     : MDEV-12729 - Timeout in include file, MDEV-13677 - Server crash
+rpl.rpl_domain_id_filter_restart      : MDEV-10684 - Wrong result
+rpl.rpl_drop_db_fail                  : MDEV-16898 - Slave fails to start
+rpl.rpl_extra_col_master_innodb       : MDEV-16570 - Extra warning
+rpl.rpl_extra_col_master_myisam       : MDEV-14203 - Extra warning
+rpl.rpl_gtid_basic                    : MDEV-10681 - server startup problem
+rpl.rpl_gtid_crash                    : MDEV-9501 - Failed registering on master, MDEV-13643 - Lost connection
+rpl.rpl_gtid_delete_domain            : MDEV-14463 - Timeout
+rpl.rpl_gtid_errorhandling            : MDEV-13261 - Crash
+rpl.rpl_gtid_mdev9033                 : MDEV-10680 - warnings
+rpl.rpl_gtid_reconnect                : MDEV-14497 - Crash
+rpl.rpl_gtid_stop_start               : MDEV-10629 - Crash on shutdown, MDEV-12629 - Valgrind warnings
+rpl.rpl_gtid_until                    : MDEV-10625 - warnings in error log
+rpl.rpl_innodb_bug30888               : MDEV-10417 - Fails on Mips
+rpl.rpl_insert                        : MDEV-9329 - Fails on Ubuntu/s390x
+rpl.rpl_insert_delayed                : MDEV-9329 - Fails on Ubuntu/s390x
+rpl.rpl_insert_id                     : MDEV-15197 - Wrong result
+rpl.rpl_insert_id_pk                  : MDEV-16567 - Assertion failure
+rpl.rpl_insert_ignore                 : MDEV-14365 - Lost connection to MySQL server during query
+rpl.rpl_invoked_features              : MDEV-10417 - Fails on Mips
+rpl.rpl_mariadb_slave_capability      : MDEV-11018 - Extra lines in binlog
+rpl.rpl_mdev6020                      : MDEV-15272 - Server crash
+rpl.rpl_mixed_implicit_commit_binlog  : Included file modified in 10.0.36
+rpl.rpl_mixed_mixing_engines          : MDEV-14489 - Sync slave with master failed
 rpl.rpl_non_direct_row_mixing_engines   : MDEV-16561 - Timeout in master_pos_wait
 rpl.rpl_non_direct_mixed_mixing_engines : MDEV-14489 - Sync slave with master failed
 rpl.rpl_non_direct_stm_mixing_engines : MDEV-14489 - Failed sync_slave_with_master
-rpl.rpl_parallel                     : MDEV-10653 - Timeouts
-rpl.rpl_parallel_conflicts           : MDEV-15272 - Server crash
-rpl.rpl_parallel_mdev6589            : MDEV-12979 - Assertion failure
-rpl.rpl_parallel_optimistic          : MDEV-15278 - Failed to sync with master; modified in 10.2.16
-rpl.rpl_parallel_optimistic_nobinlog : MDEV-15278 - Failed to sync with master
-rpl.rpl_parallel_retry               : MDEV-11119 - Crash
-rpl.rpl_parallel_temptable           : MDEV-10356 - Crash
-rpl.rpl_row_basic_2myisam            : MDEV-13875 - command "diff_files" failed
-rpl.rpl_row_drop_create_temp_table   : MDEV-14487 - Wrong result
-rpl.rpl_row_img_eng_min              : MDEV-13875 - diff_files failed
-rpl.rpl_row_img_eng_noblob           : MDEV-13875 - command "diff_files" failed
-rpl.rpl_row_index_choice             : MDEV-15196 - Slave crash
-rpl.rpl_row_mixing_engines           : MDEV-16515 - Assertion failure
-rpl.rpl_semi_sync                    : MDEV-11220 - Wrong result
-rpl.rpl_semi_sync_after_sync         : MDEV-14366 - Wrong result
-rpl.rpl_semi_sync_after_sync_row     : MDEV-14366 - Wrong result
-rpl.rpl_semi_sync_uninstall_plugin   : MDEV-7140 - Assorted failures
-rpl.rpl_set_statement_default_master : MDEV-13258 - Extra warning
-rpl.rpl_show_slave_hosts             : MDEV-10681 - Crash
-rpl.rpl_skip_replication             : MDEV-13258 - Extra warning
-rpl.rpl_slave_grp_exec               : MDEV-10514 - Deadlock
-rpl.rpl_slave_load_tmpdir_not_exist  : MDEV-14203 - Extra warning
-rpl.rpl_slow_query_log               : MDEV-13250 - Test abort
-rpl.rpl_sp_effects                   : MDEV-13249 - Crash
-rpl.rpl_start_stop_slave             : MDEV-13567 - Sync slave timeout
-rpl.rpl_mixed_implicit_commit_binlog : Include file modified in 10.2.15
-rpl.rpl_row_implicit_commit_binlog   : Include file modified in 10.2.15
-rpl.rpl_stm_implicit_commit_binlog   : Include file modified in 10.2.15
-rpl.rpl_stm_mixing_engines           : MDEV-14489 - Sync slave with master failed
-rpl.rpl_stm_multi_query              : MDEV-9501 - Failed registering on master
-rpl.rpl_stm_relay_ign_space          : MDEV-14360 - Test assertion
-rpl.rpl_stm_stop_middle_group        : MDEV-13791 - Server crash
-rpl.rpl_sync                         : MDEV-13830 - Assertion failure
+rpl.rpl_parallel                      : MDEV-10653 - Timeouts
+rpl.rpl_parallel_conflicts            : MDEV-15272 - Server crash
+rpl.rpl_parallel_mdev6589             : MDEV-12979 - Assertion failure
+rpl.rpl_parallel_multilevel2          : MDEV-14723 - Timeout
+rpl.rpl_parallel_optimistic           : MDEV-15278 - Failed to sync with master; modified in 10.2.16
+rpl.rpl_parallel_optimistic_nobinlog  : MDEV-15278 - Failed to sync with master
+rpl.rpl_parallel_retry                : MDEV-11119 - Crash
+rpl.rpl_parallel_temptable            : MDEV-10356 - Crash
+rpl.rpl_partition_innodb              : MDEV-10417 - Fails on Mips
+rpl.rpl_password_boundaries           : MDEV-11534 - Slave IO warnings
+rpl.rpl_row_001                       : MDEV-16653 - Internal check fails; modified in 10.2.17
+rpl.rpl_row_basic_11bugs              : MDEV-12171 - Server failed to start
+rpl.rpl_row_basic_2myisam             : MDEV-13875 - command "diff_files" failed
+rpl.rpl_row_drop_create_temp_table    : MDEV-14487 - Wrong result
+rpl.rpl_row_img_blobs                 : MDEV-13875 - command "diff_files" failed
+rpl.rpl_row_img_eng_min               : MDEV-13875 - diff_files failed
+rpl.rpl_row_img_eng_noblob            : MDEV-13875 - command "diff_files" failed
+rpl.rpl_row_implicit_commit_binlog    : Included file modified in 10.0.36
+rpl.rpl_row_index_choice              : MDEV-15196 - Slave crash
+rpl.rpl_row_sp001                     : MDEV-9329 - Fails on Ubuntu/s390x
+rpl.rpl_semi_sync                     : MDEV-11220 - Wrong result
+rpl.rpl_semi_sync_after_sync          : MDEV-14366 - Wrong result
+rpl.rpl_semi_sync_after_sync_row      : MDEV-14366 - Wrong result
+rpl.rpl_semi_sync_event_after_sync    : MDEV-11806 - warnings
+rpl.rpl_semi_sync_uninstall_plugin    : MDEV-7140 - Assorted failures
+rpl.rpl_semi_sync_wait_point          : MDEV-11807 - timeout in wait condition
+rpl.rpl_set_statement_default_master  : MDEV-13258 - Extra warning
+rpl.rpl_show_slave_hosts              : MDEV-10681 - Crash
+rpl.rpl_skip_replication              : MDEV-13258 - Extra warning
+rpl.rpl_slave_grp_exec                : MDEV-10514 - Deadlock
+rpl.rpl_slave_load_tmpdir_not_exist   : MDEV-14203 - Extra warning
+rpl.rpl_slow_query_log                : MDEV-13250 - Test abort
+rpl.rpl_sp_effects                    : MDEV-13249 - Crash
+rpl.rpl_start_stop_slave              : MDEV-13567 - Sync slave timeout
+rpl.rpl_stm_000001                    : Modified in 10.2.17
+rpl.rpl_stm_implicit_commit_binlog    : Included file modified in 10.0.36
+rpl.rpl_stm_mixing_engines            : MDEV-14489 - Sync slave with master failed
+rpl.rpl_stm_multi_query               : MDEV-9501 - Failed registering on master
+rpl.rpl_stm_relay_ign_space           : MDEV-14360 - Test assertion
+rpl.rpl_stm_stop_middle_group         : MDEV-13791 - Server crash
+rpl.rpl_sync                          : MDEV-13830 - Assertion failure
 rpl.rpl_temporal_mysql56_to_mariadb53 : MDEV-9501 - Failed registering on master
-rpl.rpl_typeconv                     : Include file modified in 10.2.15
-rpl.rpl_upgrade_master_info          : MDEV-11620 - Table marked as crashed
-rpl.sec_behind_master-5114           : MDEV-13878 - Wrong result
+rpl.rpl_temporary_error2              : MDEV-10634 - Wrong number of retries
+rpl.sec_behind_master-5114            : MDEV-13878 - Wrong result
 
 rpl/extra/rpl_tests.*                : MDEV-10994 - Not maintained
 
 #----------------------------------------------------------------
 
+spider.*                           : MDEV-9329 - tests are too memory-consuming
+
 spider.basic_sql                   : MDEV-11186 - Internal check fails
 
 spider/bg.direct_aggregate         : MDEV-7098 - Packets out of order
+spider/bg.direct_aggregate_part    : MDEV-7098 - Trying to unlock mutex that wasn't locked
+spider/bg.ha                       : MDEV-9329 - failures on s390x
+spider/bg.ha_part                  : MDEV-9329 - Fails on Ubuntu/s390x
+spider/bg.spider_fixes             : MDEV-7098 -Mutex problem, MDEV-9329 - failures on s390x
+spider/bg.spider_fixes_part        : MDEV-7098 - Trying to unlock mutex that wasn't locked
 spider/bg.spider3_fixes            : MDEV-12639 - Syntax error
+spider/bg.vp_fixes                 : MDEV-9329 - Fails on Ubuntu/s390x
 
-spider/handler.*                   : MDEV-10990 - Not maintained
+spider/handler.*                   : MDEV-10987, MDEV-10990 - Tests have not been maintained
 
 #----------------------------------------------------------------
 
-sphinx.sphinx                      : MDEV-10986 - Sporadic failures
-sphinx.union-5539                  : MDEV-10986 - Sporadic failures
+sphinx.*                           : MDEV-10986 - Tests have not been maintained
 
 #----------------------------------------------------------------
 
-storage_engine.*                   : Not always timely maintained
+storage_engine*                    : Tests are not always timely maintained
 
 #----------------------------------------------------------------
 
+stress.ddl_innodb                  : MDEV-10635 - Testcase timeout
+
+#----------------------------------------------------------------
+
+sys_vars.autocommit_func2                      : MDEV-9329 - Fails on Ubuntu/s390x
 sys_vars.innodb_buffer_pool_dump_at_shutdown_basic : MDEV-14280 - Unexpected error
-sys_vars.max_prepared_stmt_count_basic       : Modified in 10.2.15
-sys_vars.maximum_basic                       : Modified in 10.2.16
-sys_vars.rpl_init_slave_func                 : MDEV-10149 - Test assertion
-sys_vars.slow_query_log_func                 : MDEV-14273 - Wrong result
-sys_vars.sysvars_innodb                      : Opt file modified in 10.2.15
-sys_vars.sysvars_server_embedded             : Opt file added in 10.2.15
-sys_vars.sysvars_server_notembedded          : Opt file added in 10.2.15
-sys_vars.thread_cache_size_func              : MDEV-11775 - Wrong result
+sys_vars.innodb_log_optimize_ddl_basic         : Added in 10.2.17
+sys_vars.keep_files_on_create_basic            : MDEV-10676 - timeout
+sys_vars.log_slow_admin_statements_func        : MDEV-12235 - Server crash
+sys_vars.maximum_basic                         : Modified in 10.2.16
+sys_vars.rpl_init_slave_func                   : MDEV-10149 - Test assertion
+sys_vars.slow_query_log_func                   : MDEV-14273 - Wrong result
+sys_vars.thread_cache_size_func                : MDEV-11775 - Wrong result
+sys_vars.wait_timeout_func                     : MDEV-12896 - Wrong result
 
 #----------------------------------------------------------------
 
-tokudb.change_column_all_1000_10   : MDEV-12640 - Lost connection
-tokudb.change_column_bin           : MDEV-12640 - Lost connection
-tokudb.change_column_char          : MDEV-12822 - Lost connection
-tokudb.dir_per_db                  : MDEV-11537 - Wrong result
-tokudb.hotindex-del-0              : MDEV-16559 - Timeout
-tokudb.hotindex-insert-0           : MDEV-15271 - Timeout
-tokudb.hotindex-insert-2           : MDEV-15271 - Timeout
-tokudb.hotindex-insert-bigchar     : MDEV-12640 - Crash
-tokudb.hotindex-update-0           : MDEV-15198 - Timeout
-tokudb.hotindex-update-1           : MDEV-12640 - Crash
-tokudb.rows-32m-rand-insert        : MDEV-12640 - Crash
-tokudb.rows-32m-seq-insert         : MDEV-12640 - Crash
-tokudb.savepoint-5                 : MDEV-15280 - Wrong result
-tokudb.type_datetime               : MDEV-15193 - Wrong result
+tokudb.change_column_all_1000_10     : MDEV-12640 - Lost connection
+tokudb.change_column_bin             : MDEV-12640 - Lost connection
+tokudb.change_column_char            : MDEV-12822 - Lost connection
+tokudb.cluster_filter                : MDEV-10678 - Wrong execution plan
+tokudb.cluster_filter_hidden         : MDEV-10678 - Wrong execution plan
+tokudb.cluster_filter_unpack_varchar : MDEV-10636 - Wrong execution plan
+tokudb.dir_per_db                    : MDEV-11537 - Wrong result
+tokudb.dir_per_db_rename_to_nonexisting_schema : MDEV-14359 - Directory not empty
+tokudb.hotindex-del-0                : MDEV-16559 - Timeout
+tokudb.hotindex-insert-0             : MDEV-15271 - Timeout
+tokudb.hotindex-insert-1             : MDEV-13870 - Lost connection to MySQL server
+tokudb.hotindex-insert-2             : MDEV-15271 - Timeout
+tokudb.hotindex-insert-bigchar       : MDEV-12640 - Crash
+tokudb.hotindex-update-0             : MDEV-15198 - Timeout
+tokudb.hotindex-update-1             : MDEV-12640 - Crash
+tokudb.locks-select-update-1         : MDEV-13406 - Lock wait timeout
+tokudb.rows-32m-rand-insert          : MDEV-12640 - Crash
+tokudb.rows-32m-seq-insert           : MDEV-12640 - Crash
+tokudb.savepoint-5                   : MDEV-15280 - Wrong result
+tokudb.type_datetime                 : MDEV-15193 - Wrong result
 
-tokudb_alter_table.hcad_all_add2   : MDEV-15269 - Timeout
+tokudb_alter_table.hcad_all_add2     : MDEV-15269 - Timeout
 
-tokudb_bugs.db917                  : Modified in 10.2.15
-tokudb_bugs.xa                     : MDEV-11804 - Lock wait timeout
+tokudb_bugs.checkpoint_lock          : MDEV-10637 - Wrong processlist output
+tokudb_bugs.checkpoint_lock_3        : MDEV-10637 - Wrong processlist output
+tokudb_bugs.frm_store                : MDEV-12823 - Valgrind
+tokudb_bugs.frm_store2               : MDEV-12823 - Valgrind
+tokudb_bugs.frm_store3               : MDEV-12823 - Valgrind
+tokudb_bugs.xa                       : MDEV-11804 - Lock wait timeout
+tokudb_bugs.xa-3                     : MDEV-16953 - Corrupt log record found
 
-tokudb_backup.*                    : MDEV-11001 - Missing include file
-tokudb_sys_vars.*                  : MDEV-11001 - Missing include file
-tokudb_rpl.*                       : MDEV-11001 - Missing include file
+tokudb_backup.*                      : MDEV-11001 - Missing include file
+tokudb_sys_vars.*                    : MDEV-11001 - Missing include file
+tokudb_rpl.*                         : MDEV-11001 - Missing include file
 
 tokudb_parts.partition_alter4_tokudb : MDEV-12640 - Lost connection
 
+rpl-tokudb.*                         : MDEV-14354 - Tests fail with tcmalloc
+
 #----------------------------------------------------------------
 
+unit.*                             : suite.pm modified in 10.1.35
+
 unit.conc_basic-t                  : MDEV-15286 - not ok 7 - test_reconnect_maxpackage
 unit.conc_misc                     : MDEV-14811 - not ok 12 - test_conc49
 unit.conc_ps_bugs                  : MDEV-13252 - not ok 44 test_bug4236
 unit.lf                            : MDEV-12897 - Signal 11 thrown
+unit.ma_test_loghandler            : MDEV-10638 - record read not ok
 unit.my_atomic                     : MDEV-15670 - Signal 11 thrown
 
 #----------------------------------------------------------------
 
 vcol.binlog                        : Added in 10.2.16
 vcol.index                         : Added in 10.2.16
-vcol.partition                     : Modified in 10.2.15
+vcol.not_supported                 : MDEV-10639 - Testcase timeout
 vcol.update                        : Modified in 10.2.16
 vcol.update_binlog                 : Added in 10.2.16
-vcol.vcol_misc                     : Modified in 10.2.16
+vcol.vcol_keys_innodb              : MDEV-10639 - Testcase timeout
+vcol.vcol_misc                     : MDEV-16651 - Wrong error message; modified in 10.2.16
 
 #----------------------------------------------------------------
 
-wsrep.binlog_format                : MDEV-11532 - Could not execute check-testcase
 wsrep.foreign_key                  : MDEV-14725 - WSREP has not yet prepared node
 wsrep.mdev_6832                    : MDEV-14195 - Check testcase failed
 wsrep.pool_of_threads              : MDEV-12234 - GLIBCXX_3.4.20 not found

From 08b91548db8c1c8363b9ca7f6629f7bb819e77c3 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Mon, 9 Jul 2018 11:05:34 +0200
Subject: [PATCH 135/164] MDEV-16495 mariadb segfaults at start on FreeBSD

don't use MY_MUTEX_INIT_FAST in constructors of statically
allocated objects.
---
 sql/semisync_master_ack_receiver.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/sql/semisync_master_ack_receiver.cc b/sql/semisync_master_ack_receiver.cc
index ac17c7de40b..fc36ee35d5d 100644
--- a/sql/semisync_master_ack_receiver.cc
+++ b/sql/semisync_master_ack_receiver.cc
@@ -43,8 +43,7 @@ Ack_receiver::Ack_receiver()
   DBUG_ENTER("Ack_receiver::Ack_receiver");
 
   m_status= ST_DOWN;
-  mysql_mutex_init(key_LOCK_ack_receiver, &m_mutex,
-                   MY_MUTEX_INIT_FAST);
+  mysql_mutex_init(key_LOCK_ack_receiver, &m_mutex, NULL);
   mysql_cond_init(key_COND_ack_receiver, &m_cond, NULL);
   m_pid= 0;
 

From a39337415d173f791b2def2bc63818134319f669 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Fri, 29 Jun 2018 11:01:55 +0200
Subject: [PATCH 136/164] MDEV-14900 Upstream 10.3 debian patches

applied (at least partially):
* armhf_mroonga_storage_fail.patch (unaligned write)
* mysqld_multi.server_lsb-header.patch (add LSB header)
* fix-spelling-errors.patch (tabxml.cpp)
* hurd_socket.patch (in Platform.pm)
* remove-systemd-obsolete-target.patch
---
 mysql-test/lib/My/Platform.pm                 |  2 ++
 .../t/innodb_fts_multiple_index.test          |  2 +-
 .../mysql-test/connect/r/xml2_mult.result     |  8 ++++----
 .../mysql-test/connect/r/xml_mult.result      |  8 ++++----
 storage/connect/tabxml.cpp                    |  2 +-
 .../lib/mrn_multiple_column_key_codec.cpp     |  3 ++-
 support-files/mariadb.service.in              |  1 -
 support-files/mariadb@.service.in             |  1 -
 support-files/mysqld_multi.server.sh          | 19 +++++++++++++++++--
 9 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/mysql-test/lib/My/Platform.pm b/mysql-test/lib/My/Platform.pm
index 110cf8a20e0..29c47028b31 100644
--- a/mysql-test/lib/My/Platform.pm
+++ b/mysql-test/lib/My/Platform.pm
@@ -112,6 +112,8 @@ sub check_socket_path_length {
   return 0 if ($^O eq 'aix');
   # See Debian bug #670722 - failing on kFreeBSD even after setting short path
   return 0 if $^O eq 'gnukfreebsd' and length $path < 40;
+  # GNU/Hurd doesn't have hostpath(), but no limitation either
+  return 0 if $^O eq 'gnu';
 
   require IO::Socket::UNIX;
 
diff --git a/mysql-test/suite/innodb_fts/t/innodb_fts_multiple_index.test b/mysql-test/suite/innodb_fts/t/innodb_fts_multiple_index.test
index 012881173e0..c8293655d1b 100644
--- a/mysql-test/suite/innodb_fts/t/innodb_fts_multiple_index.test
+++ b/mysql-test/suite/innodb_fts/t/innodb_fts_multiple_index.test
@@ -25,7 +25,7 @@ ALTER TABLE t1 ADD FULLTEXT INDEX idx_1 (a);
 ALTER TABLE t1 ADD FULLTEXT INDEX idx_2 (b);
 EVAL SHOW CREATE TABLE t1;
 
-# check mutiple index with transaction
+# check multiple index with transaction
 START TRANSACTION;
 # Insert rows
 INSERT INTO t1 (a,b) VALUES
diff --git a/storage/connect/mysql-test/connect/r/xml2_mult.result b/storage/connect/mysql-test/connect/r/xml2_mult.result
index a9592e986c0..87d1118edd5 100644
--- a/storage/connect/mysql-test/connect/r/xml2_mult.result
+++ b/storage/connect/mysql-test/connect/r/xml2_mult.result
@@ -51,14 +51,14 @@ WEB	XQuery Kick Start	en	Per Bothner	2003	49.99
 WEB	XQuery Kick Start	en	Kurt Cagle	2003	49.99
 WEB	Learning XML	en	Erik T. Ray	2003	39.95
 Warnings:
-Warning	1105	Mutiple values limited to 3
+Warning	1105	Multiple values limited to 3
 # One line lost because the where clause is applied only on the first 3 rows
 SELECT category, title, author, price FROM bookstore WHERE author LIKE 'J%';
 category	title	author	price
 CHILDREN	Harry Potter	J K. Rowling	29.99
 WEB	XQuery Kick Start	James McGovern	49.99
 Warnings:
-Warning	1105	Mutiple values limited to 3
+Warning	1105	Multiple values limited to 3
 #
 # Testing concatenated values
 #
@@ -91,12 +91,12 @@ CHILDREN	Harry Potter	en	J K. Rowling	2005	29.99
 WEB	XQuery Kick Start	en	James McGovern, Per Bothner, Kurt Cagle, James Linn	2003	49.99
 WEB	Learning XML	en	Erik T. Ray	2003	39.95
 Warnings:
-Warning	1105	Mutiple values limited to 4
+Warning	1105	Multiple values limited to 4
 # The where clause is applied on the concatenated column result
 SELECT category, title, author, price FROM bookstore WHERE author LIKE 'J%';
 category	title	author	price
 CHILDREN	Harry Potter	J K. Rowling	29.99
 WEB	XQuery Kick Start	James McGovern, Per Bothner, Kurt Cagle, James Linn	49.99
 Warnings:
-Warning	1105	Mutiple values limited to 4
+Warning	1105	Multiple values limited to 4
 DROP TABLE bookstore;
diff --git a/storage/connect/mysql-test/connect/r/xml_mult.result b/storage/connect/mysql-test/connect/r/xml_mult.result
index d89debadfab..9922b40060c 100644
--- a/storage/connect/mysql-test/connect/r/xml_mult.result
+++ b/storage/connect/mysql-test/connect/r/xml_mult.result
@@ -49,14 +49,14 @@ WEB	XQuery Kick Start	en	Per Bothner	2003	49.99
 WEB	XQuery Kick Start	en	Kurt Cagle	2003	49.99
 WEB	Learning XML	en	Erik T. Ray	2003	39.95
 Warnings:
-Warning	1105	Mutiple values limited to 3
+Warning	1105	Multiple values limited to 3
 # One line lost because the where clause is applied only on the first 3 rows
 SELECT category, title, author, price FROM bookstore WHERE author LIKE 'J%';
 category	title	author	price
 CHILDREN	Harry Potter	J K. Rowling	29.99
 WEB	XQuery Kick Start	James McGovern	49.99
 Warnings:
-Warning	1105	Mutiple values limited to 3
+Warning	1105	Multiple values limited to 3
 #
 # Testing concatenated values
 #
@@ -89,12 +89,12 @@ CHILDREN	Harry Potter	en	J K. Rowling	2005	29.99
 WEB	XQuery Kick Start	en	James McGovern, Per Bothner, Kurt Cagle, James Linn	2003	49.99
 WEB	Learning XML	en	Erik T. Ray	2003	39.95
 Warnings:
-Warning	1105	Mutiple values limited to 4
+Warning	1105	Multiple values limited to 4
 # The where clause is applied on the concatenated column result
 SELECT category, title, author, price FROM bookstore WHERE author LIKE 'J%';
 category	title	author	price
 CHILDREN	Harry Potter	J K. Rowling	29.99
 WEB	XQuery Kick Start	James McGovern, Per Bothner, Kurt Cagle, James Linn	49.99
 Warnings:
-Warning	1105	Mutiple values limited to 4
+Warning	1105	Multiple values limited to 4
 DROP TABLE bookstore;
diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp
index c96e0844497..16bc9f91017 100644
--- a/storage/connect/tabxml.cpp
+++ b/storage/connect/tabxml.cpp
@@ -1845,7 +1845,7 @@ void XMULCOL::ReadColumn(PGLOBAL g)
 
       if (N > Tdbp->Limit) {
         N = Tdbp->Limit;
-        sprintf(g->Message, "Mutiple values limited to %d", Tdbp->Limit);
+        sprintf(g->Message, "Multiple values limited to %d", Tdbp->Limit);
         PushWarning(g, Tdbp);
         } // endif N
 
diff --git a/storage/mroonga/lib/mrn_multiple_column_key_codec.cpp b/storage/mroonga/lib/mrn_multiple_column_key_codec.cpp
index b2e0e0444b8..9c56997d6d8 100644
--- a/storage/mroonga/lib/mrn_multiple_column_key_codec.cpp
+++ b/storage/mroonga/lib/mrn_multiple_column_key_codec.cpp
@@ -624,9 +624,10 @@ namespace mrn {
     long long int long_long_value;
     mrn_byte_order_network_to_host(&long_long_value, grn_key, grn_key_size);
     int max_bit = (grn_key_size * 8 - 1);
-    *((long long int *)mysql_key) =
+    long_long_value =
       long_long_value ^ (((long_long_value ^ (1LL << max_bit)) >> max_bit) |
                          (1LL << max_bit));
+    memcpy(mysql_key, &long_long_value, sizeof(long_long_value));
     DBUG_VOID_RETURN;
   }
 
diff --git a/support-files/mariadb.service.in b/support-files/mariadb.service.in
index ef9fa5c2a22..8d43b6db428 100644
--- a/support-files/mariadb.service.in
+++ b/support-files/mariadb.service.in
@@ -17,7 +17,6 @@ Description=MariaDB @VERSION@ database server
 Documentation=man:mysqld(8)
 Documentation=https://mariadb.com/kb/en/library/systemd/
 After=network.target
-After=syslog.target
 
 [Install]
 WantedBy=multi-user.target
diff --git a/support-files/mariadb@.service.in b/support-files/mariadb@.service.in
index 465a0d94c62..a2f5cff0828 100644
--- a/support-files/mariadb@.service.in
+++ b/support-files/mariadb@.service.in
@@ -22,7 +22,6 @@ Description=MariaDB @VERSION@ database server (multi-instance)
 Documentation=man:mysqld(8)
 Documentation=https://mariadb.com/kb/en/library/systemd/
 After=network.target
-After=syslog.target
 
 ConditionPathExists=@sysconf2dir@/my%I.cnf
 
diff --git a/support-files/mysqld_multi.server.sh b/support-files/mysqld_multi.server.sh
index 75908a5db9d..f00c6a56af1 100644
--- a/support-files/mysqld_multi.server.sh
+++ b/support-files/mysqld_multi.server.sh
@@ -14,8 +14,23 @@
 # Version 1.0
 #
 
-basedir=/usr/local/mysql
-bindir=/usr/local/mysql/bin
+### BEGIN INIT INFO
+# Provides:          mysqld_multi
+# Required-Start: $local_fs $network $remote_fs
+# Should-Start: ypbind nscd ldap ntpd xntpd
+# Required-Stop: $local_fs $network $remote_fs
+# Default-Start:  2 3 4 5
+# Default-Stop: 0 1 6
+# Short-Description: Start and stop multiple mysql database server daemon instances
+# Description:       Controls multiple MariaDB database server daemon instances
+### END INIT INFO
+
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+NAME=mysqld_multi
+DESC=mysqld_multi
+
+basedir=/usr
+bindir=/usr/bin
 
 if test -x $bindir/mysqld_multi
 then

From aa19eda7dac2f91da76fa1783fe4bb860e59fd94 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Fri, 6 Jul 2018 04:00:49 +0200
Subject: [PATCH 137/164] MDEV-16662 CMake warnings: CMP0007

---
 cmake/for_clients.cmake | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/cmake/for_clients.cmake b/cmake/for_clients.cmake
index e5916c56ddc..2c2c965df64 100644
--- a/cmake/for_clients.cmake
+++ b/cmake/for_clients.cmake
@@ -11,9 +11,6 @@ SET(LIBS "")
 IF(POLICY CMP0011)
   CMAKE_POLICY(SET CMP0011 NEW)
 ENDIF()
-IF(POLICY CMP0007)
-  CMAKE_POLICY(SET CMP0007 OLD)
-ENDIF()
 
 # Extract dependencies using CMake's internal ${target}_LIB_DEPENDS variable
 # returned string in ${var} is can be passed to linker's command line

From 96b8909062c0b4df654f11057905b1e31398f88f Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Fri, 6 Jul 2018 04:03:43 +0200
Subject: [PATCH 138/164] MDEV-16662 CMake warnings: CMP0022

---
 CMakeLists.txt       | 6 ++----
 cmake/libutils.cmake | 3 +--
 sql/CMakeLists.txt   | 3 +--
 3 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5d95bb568cf..94e38658697 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -14,12 +14,10 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8.3)
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7)
 
-# explicitly set the policy to OLD
-# (cannot use NEW, not everyone is on cmake-2.8.12 yet)
 IF(POLICY CMP0022)
-  CMAKE_POLICY(SET CMP0022 OLD)
+  CMAKE_POLICY(SET CMP0022 NEW)
 ENDIF()
 
 # We use the LOCATION target property (CMP0026)
diff --git a/cmake/libutils.cmake b/cmake/libutils.cmake
index a31fe0d4510..621f6ecb535 100644
--- a/cmake/libutils.cmake
+++ b/cmake/libutils.cmake
@@ -269,7 +269,7 @@ MACRO(MERGE_LIBRARIES)
     IF (ARG_SOVERSION)
       SET_TARGET_PROPERTIES(${TARGET} PROPERTIES SOVERSION  ${ARG_VERSION})
     ENDIF()
-    TARGET_LINK_LIBRARIES(${TARGET} ${LIBS})
+    TARGET_LINK_LIBRARIES(${TARGET} LINK_PRIVATE ${LIBS})
     IF(ARG_OUTPUT_NAME)
       SET_TARGET_PROPERTIES(${TARGET} PROPERTIES OUTPUT_NAME "${ARG_OUTPUT_NAME}")
     ENDIF()
@@ -282,7 +282,6 @@ MACRO(MERGE_LIBRARIES)
     ENDIF()
     MYSQL_INSTALL_TARGETS(${TARGET} DESTINATION "${INSTALL_LIBDIR}" ${COMP})
   ENDIF()
-  SET_TARGET_PROPERTIES(${TARGET} PROPERTIES LINK_INTERFACE_LIBRARIES "")
   IF(ARG_SHARED AND LINK_FLAG_NO_UNDEFINED)
     # Do not allow undefined symbols in shared libraries
     GET_TARGET_PROPERTY(TARGET_LINK_FLAGS ${TARGET} LINK_FLAGS)
diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt
index 0037ff23153..c736b0e82aa 100644
--- a/sql/CMakeLists.txt
+++ b/sql/CMakeLists.txt
@@ -307,8 +307,7 @@ TARGET_LINK_LIBRARIES(mysqld sql)
 # Provide plugins with minimal set of libraries
 SET(INTERFACE_LIBS ${LIBRT})
 IF(INTERFACE_LIBS)
- SET_TARGET_PROPERTIES(mysqld PROPERTIES LINK_INTERFACE_LIBRARIES 
-  "${INTERFACE_LIBS}")
+  TARGET_LINK_LIBRARIES(mysqld LINK_PUBLIC ${INTERFACE_LIBS})
 ENDIF()
 
 # On Solaris, some extra effort is required in order to get dtrace probes

From 6b53f9d781cc19cbec96c3eb048e6407021685a2 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Fri, 6 Jul 2018 04:03:58 +0200
Subject: [PATCH 139/164] MDEV-16662 CMake warnings: CMP0026

---
 .gitignore                                    |  1 -
 CMakeLists.txt                                | 14 ++--
 ...al_db.cmake.in => create_initial_db.cmake} | 20 ++---
 cmake/install_macros.cmake                    | 16 ++--
 cmake/libutils.cmake                          | 47 +++++-------
 ...nix.cmake.in => merge_archives_unix.cmake} | 11 +--
 libmariadb                                    |  2 +-
 libmysqld/CMakeLists.txt                      |  3 +-
 pcre/CMakeLists.txt                           | 63 +++++-----------
 pcre/pcre_grep_test.sh                        | 13 ++++
 pcre/pcre_test.bat                            |  9 +++
 pcre/pcre_test.sh                             | 11 +++
 sql/CMakeLists.txt                            | 25 +++----
 storage/connect/CMakeLists.txt                |  9 +--
 .../cmake_modules/TokuMergeLibs.cmake         | 16 ++--
 win/packaging/CMakeLists.txt                  | 73 +++++++++++++++----
 .../{create_msi.cmake.in => create_msi.cmake} | 54 +++-----------
 win/packaging/extra.wxs.in                    |  4 +-
 win/packaging/heidisql.cmake                  |  6 --
 win/upgrade_wizard/CMakeLists.txt             |  3 +-
 20 files changed, 178 insertions(+), 222 deletions(-)
 rename cmake/{create_initial_db.cmake.in => create_initial_db.cmake} (79%)
 rename cmake/{merge_archives_unix.cmake.in => merge_archives_unix.cmake} (86%)
 create mode 100644 pcre/pcre_grep_test.sh
 create mode 100644 pcre/pcre_test.bat
 create mode 100644 pcre/pcre_test.sh
 rename win/packaging/{create_msi.cmake.in => create_msi.cmake} (85%)

diff --git a/.gitignore b/.gitignore
index 832df90347b..0b08870f001 100644
--- a/.gitignore
+++ b/.gitignore
@@ -94,7 +94,6 @@ packaging/rpm-oel/mysql.spec
 packaging/rpm-uln/mysql.10.0.11.spec
 packaging/solaris/postinstall-solaris
 pcre/config.h
-pcre/pcre*test.sh
 pcre/pcre.h
 pcre/pcre_chartables.c
 pcre/pcregrep
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 94e38658697..0cd4028ac75 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -19,15 +19,11 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7)
 IF(POLICY CMP0022)
   CMAKE_POLICY(SET CMP0022 NEW)
 ENDIF()
-
-# We use the LOCATION target property (CMP0026)
-# and get_target_property() for non-existent targets (CMP0045)
-# and INSTALL_NAME_DIR (CMP0042)
-IF(CMAKE_VERSION VERSION_EQUAL "3.0.0" OR
-   CMAKE_VERSION VERSION_GREATER "3.0.0")
- CMAKE_POLICY(SET CMP0026 OLD)
- CMAKE_POLICY(SET CMP0045 OLD)
- CMAKE_POLICY(SET CMP0042 OLD)
+IF(POLICY CMP0042)
+  CMAKE_POLICY(SET CMP0042 OLD)
+ENDIF()
+IF(POLICY CMP0045)
+  CMAKE_POLICY(SET CMP0045 OLD)
 ENDIF()
 IF(POLICY CMP0054)
   CMAKE_POLICY(SET CMP0054 NEW)
diff --git a/cmake/create_initial_db.cmake.in b/cmake/create_initial_db.cmake
similarity index 79%
rename from cmake/create_initial_db.cmake.in
rename to cmake/create_initial_db.cmake
index baa48847815..df972cf379a 100644
--- a/cmake/create_initial_db.cmake.in
+++ b/cmake/create_initial_db.cmake
@@ -14,11 +14,6 @@
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA 
 
 # This script creates initial database for packaging on Windows
-SET(CMAKE_SOURCE_DIR "@CMAKE_SOURCE_DIR@")
-SET(CMAKE_CURRENT_BINARY_DIR "@CMAKE_CURRENT_BINARY_DIR@")
-SET(MYSQLD_EXECUTABLE "@MYSQLD_EXECUTABLE@")
-SET(CMAKE_CFG_INTDIR "@CMAKE_CFG_INTDIR@")
-SET(WIN32 "@WIN32@")
 # Force Visual Studio to output to stdout
 IF(ENV{VS_UNICODE_OUTPUT})
  SET ($ENV{VS_UNICODE_OUTPUT})
@@ -32,30 +27,27 @@ ENDIF()
 # Create bootstrapper SQL script
 FILE(WRITE bootstrap.sql "use mysql;\n" )
 FOREACH(FILENAME mysql_system_tables.sql mysql_system_tables_data.sql mysql_performance_tables.sql)
-   FILE(STRINGS ${CMAKE_SOURCE_DIR}/scripts/${FILENAME} CONTENTS)
+   FILE(STRINGS ${TOP_SRCDIR}/scripts/${FILENAME} CONTENTS)
    FOREACH(STR ${CONTENTS})
     IF(NOT STR MATCHES "@current_hostname")
       FILE(APPEND bootstrap.sql "${STR}\n")
     ENDIF()
   ENDFOREACH()
 ENDFOREACH()
-FILE(READ ${CMAKE_SOURCE_DIR}/scripts/fill_help_tables.sql CONTENTS)
+FILE(READ ${TOP_SRCDIR}/scripts/fill_help_tables.sql CONTENTS)
 FILE(APPEND bootstrap.sql "${CONTENTS}")
 
 FILE(REMOVE_RECURSE mysql performance_schema)
 FILE(REMOVE ibdata1 ib_logfile0 ib_logfile1)
 
 MAKE_DIRECTORY(mysql)
-IF(WIN32)
-  SET(CONSOLE --console)
-ENDIF()
 
 SET(BOOTSTRAP_COMMAND 
   ${MYSQLD_EXECUTABLE} 
   --no-defaults 
-  ${CONSOLE}
+  --console
   --bootstrap 
-  --lc-messages-dir=${CMAKE_CURRENT_BINARY_DIR}/share
+  --lc-messages-dir=${BINDIR}/share
   --basedir=.
   --datadir=.
   --default-storage-engine=MyISAM
@@ -65,10 +57,10 @@ SET(BOOTSTRAP_COMMAND
 
 GET_FILENAME_COMPONENT(CWD . ABSOLUTE)
 EXECUTE_PROCESS(
-  COMMAND "@CMAKE_COMMAND@" -E echo Executing ${BOOTSTRAP_COMMAND}
+  COMMAND "${CMAKE_COMMAND}" -E echo Executing ${BOOTSTRAP_COMMAND}
 )
 EXECUTE_PROCESS (
-  COMMAND "@CMAKE_COMMAND@" -E
+  COMMAND "${CMAKE_COMMAND}" -E
   echo input file bootstrap.sql, current directory ${CWD}
 )
 EXECUTE_PROCESS (  
diff --git a/cmake/install_macros.cmake b/cmake/install_macros.cmake
index d9d12371830..71657e32308 100644
--- a/cmake/install_macros.cmake
+++ b/cmake/install_macros.cmake
@@ -159,21 +159,17 @@ ENDFUNCTION()
 
 
 # Install symbolic link to CMake target. 
-# the link is created in the same directory as target
+# the link is created in the current build directory
 # and extension will be the same as for target file.
 MACRO(INSTALL_SYMLINK linkname target destination component)
 IF(UNIX)
-  GET_TARGET_PROPERTY(location ${target} LOCATION)
-  GET_FILENAME_COMPONENT(path ${location} PATH)
-  GET_FILENAME_COMPONENT(name ${location} NAME)
-  SET(output ${path}/${linkname})
+  SET(output ${CMAKE_CURRENT_BINARY_DIR}/${linkname})
   ADD_CUSTOM_COMMAND(
     OUTPUT ${output}
-    COMMAND ${CMAKE_COMMAND} ARGS -E remove -f ${output}
+    COMMAND ${CMAKE_COMMAND} ARGS -E remove -f ${linkname}
     COMMAND ${CMAKE_COMMAND} ARGS -E create_symlink 
-      ${name} 
+      $<TARGET_FILE_NAME:${target}>
       ${linkname}
-    WORKING_DIRECTORY ${path}
     DEPENDS ${target}
     )
   
@@ -259,8 +255,7 @@ FUNCTION(MYSQL_INSTALL_TARGETS)
     ENDIF()
     # Install man pages on Unix
     IF(UNIX)
-      GET_TARGET_PROPERTY(target_location ${target} LOCATION)
-      INSTALL_MANPAGE(${target_location})
+      INSTALL_MANPAGE($<TARGET_FILE:${target}>)
     ENDIF()
   ENDFOREACH()
 
@@ -277,6 +272,7 @@ SET(DEBUGBUILDDIR "${BINARY_PARENTDIR}/debug" CACHE INTERNAL "Directory of debug
 
 
 FUNCTION(INSTALL_DEBUG_TARGET target)
+  RETURN() # XXX unused?
   CMAKE_PARSE_ARGUMENTS(ARG
   ""
   "DESTINATION;RENAME;PDB_DESTINATION;COMPONENT"
diff --git a/cmake/libutils.cmake b/cmake/libutils.cmake
index 621f6ecb535..cdc7af5d228 100644
--- a/cmake/libutils.cmake
+++ b/cmake/libutils.cmake
@@ -139,17 +139,17 @@ MACRO(MERGE_STATIC_LIBS TARGET OUTPUT_NAME LIBS_TO_MERGE)
 
   SET(OSLIBS)
   FOREACH(LIB ${LIBS_TO_MERGE})
-    GET_TARGET_PROPERTY(LIB_LOCATION ${LIB} LOCATION)
-    GET_TARGET_PROPERTY(LIB_TYPE ${LIB} TYPE)
-    IF(NOT LIB_LOCATION)
+    IF(NOT TARGET ${LIB})
        # 3rd party library like libz.so. Make sure that everything
        # that links to our library links to this one as well.
        LIST(APPEND OSLIBS ${LIB})
     ELSE()
+      GET_TARGET_PROPERTY(LIB_TYPE ${LIB} TYPE)
       # This is a target in current project
       # (can be a static or shared lib)
       IF(LIB_TYPE STREQUAL "STATIC_LIBRARY")
-        SET(STATIC_LIBS ${STATIC_LIBS} ${LIB_LOCATION})
+        SET(STATIC_TGTS ${STATIC_TGTS} ${LIB})
+        SET(STATIC_LIBS ${STATIC_LIBS} $<TARGET_FILE:${LIB}>)
         ADD_DEPENDENCIES(${TARGET} ${LIB})
         # Extract dependent OS libraries
         GET_DEPENDEND_OS_LIBS(${LIB} LIB_OSLIBS)
@@ -171,7 +171,7 @@ MACRO(MERGE_STATIC_LIBS TARGET OUTPUT_NAME LIBS_TO_MERGE)
   ADD_CUSTOM_COMMAND( 
     OUTPUT  ${SOURCE_FILE}
     COMMAND ${CMAKE_COMMAND}  -E touch ${SOURCE_FILE}
-    DEPENDS ${STATIC_LIBS})
+    DEPENDS ${STATIC_TGTS})
 
   IF(MSVC)
     # To merge libs, just pass them to lib.exe command line.
@@ -182,29 +182,27 @@ MACRO(MERGE_STATIC_LIBS TARGET OUTPUT_NAME LIBS_TO_MERGE)
     SET_TARGET_PROPERTIES(${TARGET} PROPERTIES STATIC_LIBRARY_FLAGS 
       "${LINKER_EXTRA_FLAGS}")
   ELSE()
-    GET_TARGET_PROPERTY(TARGET_LOCATION ${TARGET} LOCATION)  
     IF(APPLE)
       # Use OSX's libtool to merge archives (ihandles universal 
       # binaries properly)
       ADD_CUSTOM_COMMAND(TARGET ${TARGET} POST_BUILD
-        COMMAND rm ${TARGET_LOCATION}
-        COMMAND libtool -static -o ${TARGET_LOCATION} 
+        COMMAND rm $<TARGET_FILE:${TARGET}>
+        COMMAND libtool -static -o $<TARGET_FILE:${TARGET}>
         ${STATIC_LIBS}
       )  
     ELSE()
       # Generic Unix, Cygwin or MinGW. In post-build step, call
       # script, that extracts objects from archives with "ar x" 
       # and repacks them with "ar r"
-      SET(TARGET ${TARGET})
-      CONFIGURE_FILE(
-        ${MYSQL_CMAKE_SCRIPT_DIR}/merge_archives_unix.cmake.in
-        ${CMAKE_CURRENT_BINARY_DIR}/merge_archives_${TARGET}.cmake 
-        @ONLY
-      )
       ADD_CUSTOM_COMMAND(TARGET ${TARGET} POST_BUILD
-        COMMAND rm ${TARGET_LOCATION}
-        COMMAND ${CMAKE_COMMAND} -P 
-        ${CMAKE_CURRENT_BINARY_DIR}/merge_archives_${TARGET}.cmake
+        COMMAND ${CMAKE_COMMAND}
+          -DTARGET_LOCATION="$<TARGET_FILE:${TARGET}>"
+          -DTARGET="${TARGET}"
+          -DSTATIC_LIBS="${STATIC_LIBS}"
+          -DCMAKE_CURRENT_BINARY_DIR="${CMAKE_CURRENT_BINARY_DIR}"
+          -DCMAKE_AR="${CMAKE_AR}"
+          -DCMAKE_RANLIB="${CMAKE_RANLIB}"
+          -P "${MYSQL_CMAKE_SCRIPT_DIR}/merge_archives_unix.cmake"
       )
     ENDIF()
   ENDIF()
@@ -294,18 +292,11 @@ MACRO(MERGE_LIBRARIES)
 ENDMACRO()
 
 FUNCTION(GET_DEPENDEND_OS_LIBS target result)
-  SET(deps ${${target}_LIB_DEPENDS})
-  IF(deps)
-   FOREACH(lib ${deps})
-    # Filter out keywords for used for debug vs optimized builds
-    IF(NOT lib MATCHES "general" AND NOT lib MATCHES "debug" AND NOT lib MATCHES "optimized")
-      GET_TARGET_PROPERTY(lib_location ${lib} LOCATION)
-      IF(NOT lib_location)
-        SET(ret ${ret} ${lib})
-      ENDIF()
+  FOREACH(lib ${${target}_LIB_DEPENDS})
+    IF(NOT TARGET ${lib})
+      SET(ret ${ret} ${lib})
     ENDIF()
-   ENDFOREACH()
-  ENDIF()
+  ENDFOREACH()
   SET(${result} ${ret} PARENT_SCOPE)
 ENDFUNCTION()
 
diff --git a/cmake/merge_archives_unix.cmake.in b/cmake/merge_archives_unix.cmake
similarity index 86%
rename from cmake/merge_archives_unix.cmake.in
rename to cmake/merge_archives_unix.cmake
index d05336d6c91..08b1931c3d3 100644
--- a/cmake/merge_archives_unix.cmake.in
+++ b/cmake/merge_archives_unix.cmake
@@ -14,19 +14,12 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA 
 
-# This script merges many static libraries into
-# one big library on Unix.
-SET(TARGET_LOCATION "@TARGET_LOCATION@")
-SET(TARGET "@TARGET@")
-SET(STATIC_LIBS "@STATIC_LIBS@")
-SET(CMAKE_CURRENT_BINARY_DIR "@CMAKE_CURRENT_BINARY_DIR@")
-SET(CMAKE_AR "@CMAKE_AR@")
-SET(CMAKE_RANLIB "@CMAKE_RANLIB@")
-
+FILE(REMOVE "${TARGET_LOCATION}")
 
 SET(TEMP_DIR ${CMAKE_CURRENT_BINARY_DIR}/merge_archives_${TARGET})
 MAKE_DIRECTORY(${TEMP_DIR})
 # Extract each archive to its own subdirectory(avoid object filename clashes)
+SEPARATE_ARGUMENTS(STATIC_LIBS UNIX_COMMAND "${STATIC_LIBS}")
 FOREACH(LIB ${STATIC_LIBS})
   GET_FILENAME_COMPONENT(NAME_NO_EXT ${LIB} NAME_WE)
   SET(TEMP_SUBDIR ${TEMP_DIR}/${NAME_NO_EXT})
diff --git a/libmariadb b/libmariadb
index ebf5db6cd0c..ed5921ed4f2 160000
--- a/libmariadb
+++ b/libmariadb
@@ -1 +1 @@
-Subproject commit ebf5db6cd0c7d10b0632a1389faa9731e3c09cf3
+Subproject commit ed5921ed4f26407ff1ff0596f2c74c8d7098632d
diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt
index d7450eb3e87..99b620882de 100644
--- a/libmysqld/CMakeLists.txt
+++ b/libmysqld/CMakeLists.txt
@@ -155,8 +155,7 @@ SET(LIBS
 # (with corresponding target ${engine}_embedded)
 SET(EMBEDDED_LIBS)
 FOREACH(LIB ${LIBS})
-  GET_TARGET_PROPERTY(EMBEDDED_LOCATION ${LIB}_embedded LOCATION)
-  IF(EMBEDDED_LOCATION)
+  IF(TARGET ${LIB}_embedded)
     LIST(APPEND EMBEDDED_LIBS ${LIB}_embedded)
   ELSE()
     LIST(APPEND EMBEDDED_LIBS ${LIB})
diff --git a/pcre/CMakeLists.txt b/pcre/CMakeLists.txt
index 80e402e5713..ca60d9ed265 100644
--- a/pcre/CMakeLists.txt
+++ b/pcre/CMakeLists.txt
@@ -364,12 +364,6 @@ IF(PCRE_BUILD_TESTS)
 
   # exes in Debug location tested by the RunTest shell script
   # via "make test"
-  IF(PCRE_BUILD_PCREGREP)
-    GET_TARGET_PROPERTY(PCREGREP_EXE pcregrep DEBUG_LOCATION)
-  ENDIF(PCRE_BUILD_PCREGREP)
-
-  GET_TARGET_PROPERTY(PCRETEST_EXE pcretest DEBUG_LOCATION)
-  STRING(REGEX REPLACE "[^/]+$" "" PCRETEST_DIR ${PCRETEST_EXE})
 
 # =================================================
   # Write out a CTest configuration file
@@ -379,38 +373,26 @@ IF(PCRE_BUILD_TESTS)
 MESSAGE(\"When testing is complete, review test output in the
 \\\"${PROJECT_BINARY_DIR}/Testing/Temporary\\\" folder.\")
 MESSAGE(\" \")
-")
-
-  FILE(WRITE ${PROJECT_BINARY_DIR}/pcre_test.sh
-  "#! /bin/sh
-# This is a generated file.
-srcdir=${PROJECT_SOURCE_DIR}
-pcretest=${PCRETEST_EXE}
-cd ${PCRETEST_DIR}
-. ${PROJECT_SOURCE_DIR}/RunTest
-if test \"$?\" != \"0\"; then exit 1; fi
-# End
 ")
 
   IF(UNIX)
-    ADD_TEST(pcre_test      sh ${PROJECT_BINARY_DIR}/pcre_test.sh)
+    ADD_TEST(NAME pcre_test
+      COMMAND sh pcre_test.sh
+        ${PROJECT_SOURCE_DIR}
+        $<TARGET_FILE:pcretest>
+        $<TARGET_FILE_DIR:pcretest>
+      )
   ENDIF(UNIX)
 
   IF(PCRE_BUILD_PCREGREP)
-    FILE(WRITE ${PROJECT_BINARY_DIR}/pcre_grep_test.sh
-    "#! /bin/sh
-# This is a generated file.
-srcdir=${PROJECT_SOURCE_DIR}
-pcregrep=${PCREGREP_EXE}
-pcretest=${PCRETEST_EXE}
-cd ${PCRETEST_DIR}
-. ${PROJECT_SOURCE_DIR}/RunGrepTest
-if test \"$?\" != \"0\"; then exit 1; fi
-# End
-")
-
     IF(UNIX)
-      ADD_TEST(pcre_grep_test sh ${PROJECT_BINARY_DIR}/pcre_grep_test.sh)
+      ADD_TEST(NAME pcre_grep_test
+        COMMAND sh pcre_grep_test.sh
+          ${PROJECT_SOURCE_DIR}
+          $<TARGET_FILE:pcregrep>
+          $<TARGET_FILE:pcretest>
+          $<TARGET_FILE_DIR:pcretest>
+        )
     ENDIF(UNIX)
   ENDIF(PCRE_BUILD_PCREGREP)
 
@@ -418,22 +400,13 @@ if test \"$?\" != \"0\"; then exit 1; fi
     # Provide environment for executing the bat file version of RunTest
     FILE(TO_NATIVE_PATH ${PROJECT_SOURCE_DIR} winsrc)
     FILE(TO_NATIVE_PATH ${PROJECT_BINARY_DIR} winbin)
-    FILE(TO_NATIVE_PATH ${PCRETEST_EXE} winexe)
-
-    FILE(WRITE ${PROJECT_BINARY_DIR}/pcre_test.bat
-    "\@REM This is a generated file.
-\@echo off
-setlocal
-SET srcdir=\"${winsrc}\"
-SET pcretest=\"${winexe}\"
-if not [%CMAKE_CONFIG_TYPE%]==[] SET pcretest=\"${winbin}\\%CMAKE_CONFIG_TYPE%\\pcretest.exe\"
-call %srcdir%\\RunTest.Bat
-if errorlevel 1 exit /b 1
-echo RunTest.bat tests successfully completed
-")
 
   ADD_TEST(NAME pcre_test_bat
-  COMMAND pcre_test.bat)
+    COMMAND pcre_test.bat
+      ${winsrc}
+      ${winbin}
+      $<TARGET_FILE:pcretest>
+    )
   SET_TESTS_PROPERTIES(pcre_test_bat PROPERTIES
   PASS_REGULAR_EXPRESSION "RunTest\\.bat tests successfully completed")
 
diff --git a/pcre/pcre_grep_test.sh b/pcre/pcre_grep_test.sh
new file mode 100644
index 00000000000..7165aeeb0e2
--- /dev/null
+++ b/pcre/pcre_grep_test.sh
@@ -0,0 +1,13 @@
+#! /bin/sh
+#
+srcdir="$1"
+pcregrep="$2"
+pcretest="$3"
+cd "$4"
+shift
+shift
+shift
+shift
+. "$srcdir"/RunGrepTest
+if test "$?" != "0"; then exit 1; fi
+# End
diff --git a/pcre/pcre_test.bat b/pcre/pcre_test.bat
new file mode 100644
index 00000000000..c3f8dc3d43b
--- /dev/null
+++ b/pcre/pcre_test.bat
@@ -0,0 +1,9 @@
+@echo off
+setlocal
+SET srcdir="%1"
+SET bindir="%1"
+SET pcretest="%3"
+if not [%CMAKE_CONFIG_TYPE%]==[] SET pcretest="%bindir%\%CMAKE_CONFIG_TYPE%\pcretest.exe"
+call %srcdir%\RunTest.Bat
+if errorlevel 1 exit /b 1
+echo RunTest.bat tests successfully completed
diff --git a/pcre/pcre_test.sh b/pcre/pcre_test.sh
new file mode 100644
index 00000000000..8fadc671c9a
--- /dev/null
+++ b/pcre/pcre_test.sh
@@ -0,0 +1,11 @@
+#! /bin/sh
+#
+srcdir="$1"
+pcretest="$2"
+cd "$3"
+shift
+shift
+shift
+. "$srcdir"/RunTest
+if test "$?" != "0"; then exit 1; fi
+# End
diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt
index c736b0e82aa..708c36a58b0 100644
--- a/sql/CMakeLists.txt
+++ b/sql/CMakeLists.txt
@@ -203,9 +203,7 @@ IF(MSVC AND NOT WITHOUT_DYNAMIC_PLUGINS)
   SET(MYSQLD_LIB ${CMAKE_CURRENT_BINARY_DIR}/mysqld_lib.lib)
   SET(MYSQLD_CORELIBS sql mysys dbug strings)
   FOREACH (CORELIB ${MYSQLD_CORELIBS})
-    GET_TARGET_PROPERTY(LOC ${CORELIB} LOCATION)
-    FILE(TO_NATIVE_PATH ${LOC} LOC)
-    SET (LIB_LOCATIONS ${LIB_LOCATIONS} ${LOC})
+    SET (LIB_LOCATIONS ${LIB_LOCATIONS} $<TARGET_FILE:${CORELIB}>)
   ENDFOREACH (CORELIB)
 
   SET(_PLATFORM x86)
@@ -420,23 +418,20 @@ SET(DBOPT_FILE ${CMAKE_SOURCE_DIR}/support-files/db.opt )
 INSTALL(FILES ${DBOPT_FILE} DESTINATION data/test COMPONENT DataFiles)
 
 # Install initial database on windows
-IF(NOT CMAKE_CROSSCOMPILING)
-  GET_TARGET_PROPERTY(MYSQLD_EXECUTABLE mysqld LOCATION)
-ENDIF()
-IF(WIN32 AND MYSQLD_EXECUTABLE)
-  CONFIGURE_FILE(
-    ${CMAKE_SOURCE_DIR}/cmake/create_initial_db.cmake.in
-    ${CMAKE_CURRENT_BINARY_DIR}/create_initial_db.cmake
-    @ONLY
-  )
-  
+IF(WIN32 AND TARGET mysqld AND NOT CMAKE_CROSSCOMPILING)
+
   IF(MSVC_IDE OR CMAKE_GENERATOR MATCHES "Xcode")
     SET (CONFIG_PARAM -DCONFIG=${CMAKE_CFG_INTDIR})
   ENDIF()
   MAKE_DIRECTORY(${CMAKE_CURRENT_BINARY_DIR}/data)
   ADD_CUSTOM_COMMAND(
      OUTPUT  ${CMAKE_CURRENT_BINARY_DIR}/initdb.dep
-     COMMAND ${CMAKE_COMMAND} ${CONFIG_PARAM} -P ${CMAKE_CURRENT_BINARY_DIR}/create_initial_db.cmake
+     COMMAND ${CMAKE_COMMAND} ${CONFIG_PARAM}
+       -DTOP_SRCDIR="${CMAKE_SOURCE_DIR}"
+       -DBINDIR="${CMAKE_CURRENT_BINARY_DIR}"
+       -DMYSQLD_EXECUTABLE="$<TARGET_FILE:mysqld>"
+       -DCMAKE_CFG_INTDIR="${CMAKE_CFG_INTDIR}"
+       -P ${CMAKE_SOURCE_DIR}/cmake/create_initial_db.cmake
      COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/initdb.dep
      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/data
      DEPENDS mysqld
@@ -454,7 +449,7 @@ IF(WIN32 AND MYSQLD_EXECUTABLE)
 ELSE()
   # Not windows or cross compiling, just install an empty directory
   INSTALL(FILES ${DUMMY_FILE} DESTINATION data/mysql COMPONENT DataFiles)
-ENDIF(WIN32 AND MYSQLD_EXECUTABLE)
+ENDIF(WIN32 AND TARGET mysqld AND NOT CMAKE_CROSSCOMPILING)
 ENDIF(INSTALL_LAYOUT STREQUAL "STANDALONE")
 
 IF(WIN32)
diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt
index 782d1f44bdf..9a08ff08425 100644
--- a/storage/connect/CMakeLists.txt
+++ b/storage/connect/CMakeLists.txt
@@ -338,14 +338,7 @@ IF(WIN32)
 
 # Install some extra files that belong to connect engine
 
-  # install ha_connect.lib
-  GET_TARGET_PROPERTY(CONNECT_LOCATION connect LOCATION)
-  STRING(REPLACE "dll" "lib" CONNECT_LIB ${CONNECT_LOCATION})
-  IF(CMAKE_CONFIGURATION_TYPES)
-    STRING(REPLACE "${CMAKE_CFG_INTDIR}" "\${CMAKE_INSTALL_CONFIG_NAME}"
-      CONNECT_LIB ${CONNECT_LIB})
-  ENDIF()
-  INSTALL(FILES ${CONNECT_LIB}
+  INSTALL(FILES "$<TARGET_FILE_DIR:connect>/ha_connect.lib"
     DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine)
 ENDIF(WIN32)
 
diff --git a/storage/tokudb/PerconaFT/cmake_modules/TokuMergeLibs.cmake b/storage/tokudb/PerconaFT/cmake_modules/TokuMergeLibs.cmake
index e1da095fc00..3413339628f 100644
--- a/storage/tokudb/PerconaFT/cmake_modules/TokuMergeLibs.cmake
+++ b/storage/tokudb/PerconaFT/cmake_modules/TokuMergeLibs.cmake
@@ -3,17 +3,11 @@
 # We use it in MySQL to merge mysys,dbug,vio etc into mysqlclient
 FUNCTION(TOKU_GET_DEPENDEND_OS_LIBS target result)
   SET(deps ${${target}_LIB_DEPENDS})
-  IF(deps)
-   FOREACH(lib ${deps})
-    # Filter out keywords for used for debug vs optimized builds
-    IF(NOT lib MATCHES "general" AND NOT lib MATCHES "debug" AND NOT lib MATCHES "optimized")
-      GET_TARGET_PROPERTY(lib_location ${lib} LOCATION)
-      IF(NOT lib_location)
-        SET(ret ${ret} ${lib})
-      ENDIF()
-    ENDIF()
-   ENDFOREACH()
-  ENDIF()
+  FOREACH(lib ${deps})
+   IF(TARGET ${lib})
+     SET(ret ${ret} ${lib})
+   ENDIF()
+  ENDFOREACH()
   SET(${result} ${ret} PARENT_SCOPE)
 ENDFUNCTION(TOKU_GET_DEPENDEND_OS_LIBS)
 
diff --git a/win/packaging/CMakeLists.txt b/win/packaging/CMakeLists.txt
index d5dc3becba8..61bfe9e5568 100644
--- a/win/packaging/CMakeLists.txt
+++ b/win/packaging/CMakeLists.txt
@@ -123,11 +123,9 @@ ELSE()
   FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/COPYING.rtf" "\n}\n")
   SET(COPYING_RTF "${CMAKE_CURRENT_BINARY_DIR}/COPYING.rtf")
 ENDIF()
-GET_TARGET_PROPERTY(WIXCA_LOCATION wixca LOCATION)
 SET(CPACK_WIX_CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/CPackWixConfig.cmake)
 
-GET_TARGET_PROPERTY(upgrade_wizard_location mysql_upgrade_wizard LOCATION)
-IF(NOT upgrade_wizard_location)
+IF(NOT TARGET mysql_upgrade_wizard)
   SET(EXTRA_WIX_PREPROCESSOR_FLAGS "-dHaveUpgradeWizard=0")
 ENDIF()
 IF(WITH_INNOBASE_STORAGE_ENGINE)
@@ -145,7 +143,6 @@ IF(WITH_THIRD_PARTY)
   ENDIF()
 ENDIF()
 
-GET_TARGET_PROPERTY(LIBMYSQL_LOCATION libmysql LOCATION)
 FOREACH(third_party ${WITH_THIRD_PARTY})
   SET(third_party_install_plugin ${CMAKE_CURRENT_SOURCE_DIR}/${third_party}.cmake)
   IF(NOT EXISTS ${third_party_install_plugin})
@@ -169,10 +166,6 @@ ENDIF()
 IF(NOT CPACK_WIX_UI)
   SET(CPACK_WIX_UI "MyWixUI_Mondo")
 ENDIF()
-CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/create_msi.cmake.in
-  ${CMAKE_CURRENT_BINARY_DIR}/create_msi.cmake
-  @ONLY)
-
 
 IF(CMAKE_SIZEOF_VOID_P EQUAL 8)
   SET(WixWin64 " Win64='yes'")
@@ -188,17 +181,69 @@ ENDIF()
 
 ADD_CUSTOM_TARGET(
   MSI
-  COMMAND ${CMAKE_COMMAND} 
-  ${CONFIG_PARAM} 
-  -P  ${CMAKE_CURRENT_BINARY_DIR}/create_msi.cmake
+  COMMAND ${CMAKE_COMMAND} ${CONFIG_PARAM}
+    -DCANDLE_EXECUTABLE="${CANDLE_EXECUTABLE}"
+    -DCMAKE_CFG_INTDIR="${CMAKE_CFG_INTDIR}"
+    -DCMAKE_FULL_VER="${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}.${CMAKE_PATCH_VERSION}"
+    -DCMAKE_SIZEOF_VOID_P=${CMAKE_SIZEOF_VOID_P}
+    -DCOPYING_RTF="${COPYING_RTF}"
+    -DCPACK_WIX_CONFIG="${CPACK_WIX_CONFIG}"
+    -DCPACK_WIX_INCLUDE="${CPACK_WIX_INCLUDE}"
+    -DCPACK_WIX_PACKAGE_BASE_NAME="${CPACK_WIX_PACKAGE_BASE_NAME}"
+    -DCPACK_WIX_PACKAGE_NAME="${CPACK_WIX_PACKAGE_NAME}"
+    -DCPACK_WIX_UPGRADE_CODE="${CPACK_WIX_UPGRADE_CODE}"
+    -DEXTRA_WIX_PREPROCESSOR_FLAGS="${EXTRA_WIX_PREPROCESSOR_FLAGS}"
+    -DLIGHT_EXECUTABLE="${LIGHT_EXECUTABLE}"
+    -DMAJOR_VERSION="${MAJOR_VERSION}"
+    -DMANUFACTURER="${MANUFACTURER}"
+    -DMINOR_VERSION="${MINOR_VERSION}"
+    -DPATCH_VERSION="${PATCH_VERSION}"
+    -DSIGNCODE="${SIGNCODE}"
+    -DSIGNTOOL_EXECUTABLE="${SIGNTOOL_EXECUTABLE}"
+    -DSIGNTOOL_PARAMETERS="${SIGNTOOL_PARAMETERS}"
+    -DSRCDIR="${CMAKE_CURRENT_SOURCE_DIR}"
+    -DTHIRD_PARTY_DOWNLOAD_LOCATION="${THIRD_PARTY_DOWNLOAD_LOCATION}"
+    -DTHIRD_PARTY_FEATURE_CONDITION="${THIRD_PARTY_FEATURE_CONDITION}"
+    -DTINY_VERSION="${TINY_VERSION}"
+    -DTOP_BINDIR="${CMAKE_BINARY_DIR}"
+    -DVERSION="${VERSION}"
+    -DWITH_THIRD_PARTY="${WITH_THIRD_PARTY}"
+    -DWIXCA_LOCATION="$<TARGET_FILE:wixca>"
+    -P  ${CMAKE_CURRENT_SOURCE_DIR}/create_msi.cmake
 )
 ADD_DEPENDENCIES(MSI wixca)
 
 ADD_CUSTOM_TARGET(
   MSI_ESSENTIALS
-  COMMAND ${CMAKE_COMMAND} -DESSENTIALS=1 
-  ${CONFIG_PARAM}
-  -P  ${CMAKE_CURRENT_BINARY_DIR}/create_msi.cmake
+  COMMAND ${CMAKE_COMMAND} ${CONFIG_PARAM} -DESSENTIALS=1
+    -DCANDLE_EXECUTABLE="${CANDLE_EXECUTABLE}"
+    -DCMAKE_CFG_INTDIR="${CMAKE_CFG_INTDIR}"
+    -DCMAKE_FULL_VER="${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}.${CMAKE_PATCH_VERSION}"
+    -DCMAKE_SIZEOF_VOID_P=${CMAKE_SIZEOF_VOID_P}
+    -DCOPYING_RTF="${COPYING_RTF}"
+    -DCPACK_WIX_CONFIG="${CPACK_WIX_CONFIG}"
+    -DCPACK_WIX_INCLUDE="${CPACK_WIX_INCLUDE}"
+    -DCPACK_WIX_PACKAGE_BASE_NAME="${CPACK_WIX_PACKAGE_BASE_NAME}"
+    -DCPACK_WIX_PACKAGE_NAME="${CPACK_WIX_PACKAGE_NAME}"
+    -DCPACK_WIX_UPGRADE_CODE="${CPACK_WIX_UPGRADE_CODE}"
+    -DEXTRA_WIX_PREPROCESSOR_FLAGS="${EXTRA_WIX_PREPROCESSOR_FLAGS}"
+    -DLIGHT_EXECUTABLE="${LIGHT_EXECUTABLE}"
+    -DMAJOR_VERSION="${MAJOR_VERSION}"
+    -DMANUFACTURER="${MANUFACTURER}"
+    -DMINOR_VERSION="${MINOR_VERSION}"
+    -DPATCH_VERSION="${PATCH_VERSION}"
+    -DSIGNCODE="${SIGNCODE}"
+    -DSIGNTOOL_EXECUTABLE="${SIGNTOOL_EXECUTABLE}"
+    -DSIGNTOOL_PARAMETERS="${SIGNTOOL_PARAMETERS}"
+    -DSRCDIR="${CMAKE_CURRENT_SOURCE_DIR}"
+    -DTHIRD_PARTY_DOWNLOAD_LOCATION="${THIRD_PARTY_DOWNLOAD_LOCATION}"
+    -DTHIRD_PARTY_FEATURE_CONDITION="${THIRD_PARTY_FEATURE_CONDITION}"
+    -DTINY_VERSION="${TINY_VERSION}"
+    -DTOP_BINDIR="${CMAKE_BINARY_DIR}"
+    -DVERSION="${VERSION}"
+    -DWITH_THIRD_PARTY="${WITH_THIRD_PARTY}"
+    -DWIXCA_LOCATION="$<TARGET_FILE:wixca>"
+    -P  ${CMAKE_CURRENT_SOURCE_DIR}/create_msi.cmake
 )
 ADD_DEPENDENCIES(MSI_ESSENTIALS wixca)
 
diff --git a/win/packaging/create_msi.cmake.in b/win/packaging/create_msi.cmake
similarity index 85%
rename from win/packaging/create_msi.cmake.in
rename to win/packaging/create_msi.cmake
index c37f7ca9d1d..f0407684ff3 100644
--- a/win/packaging/create_msi.cmake.in
+++ b/win/packaging/create_msi.cmake
@@ -1,35 +1,3 @@
-SET(CMAKE_BINARY_DIR "@CMAKE_BINARY_DIR@")
-SET(CMAKE_CURRENT_SOURCE_DIR "@CMAKE_CURRENT_SOURCE_DIR@")
-SET(CANDLE_EXECUTABLE "@CANDLE_EXECUTABLE@")
-SET(LIGHT_EXECUTABLE "@LIGHT_EXECUTABLE@")
-SET(CMAKE_COMMAND "@CMAKE_COMMAND@")
-SET(CMAKE_CFG_INTDIR "@CMAKE_CFG_INTDIR@")
-SET(VERSION "@VERSION@")
-SET(MAJOR_VERSION "@MAJOR_VERSION@")
-SET(MINOR_VERSION "@MINOR_VERSION@")
-SET(PATCH_VERSION "@PATCH_VERSION@")
-SET(TINY_VERSION "@TINY_VERSION@")
-SET(CMAKE_SIZEOF_VOID_P @CMAKE_SIZEOF_VOID_P@)
-SET(MANUFACTURER "@MANUFACTURER@")
-SET(WIXCA_LOCATION "@WIXCA_LOCATION@")
-SET(COPYING_RTF "@COPYING_RTF@")
-SET(CPACK_WIX_CONFIG "@CPACK_WIX_CONFIG@")
-SET(CPACK_WIX_INCLUDE "@CPACK_WIX_INCLUDE@")
-SET(CPACK_WIX_UPGRADE_CODE "@CPACK_WIX_UPGRADE_CODE@")
-SET(CPACK_WIX_PACKAGE_NAME "@CPACK_WIX_PACKAGE_NAME@")
-SET(CPACK_WIX_PACKAGE_BASE_NAME "@CPACK_WIX_PACKAGE_BASE_NAME@")
-SET(SIGNCODE "@SIGNCODE@")
-SET(SIGNTOOL_EXECUTABLE "@SIGNTOOL_EXECUTABLE@")
-SET(SIGNTOOL_PARAMETERS "@SIGNTOOL_PARAMETERS@")
-SET(CMAKE_FULL_VER 
-  "@CMAKE_MAJOR_VERSION@.@CMAKE_MINOR_VERSION@.@CMAKE_PATCH_VERSION@")
-SET(EXTRA_WIX_PREPROCESSOR_FLAGS  "@EXTRA_WIX_PREPROCESSOR_FLAGS@")
-SET(WITH_THIRD_PARTY "@WITH_THIRD_PARTY@")
-SET(THIRD_PARTY_DOWNLOAD_LOCATION "@THIRD_PARTY_DOWNLOAD_LOCATION@")
-SET(THIRD_PARTY_FEATURE_CONDITION "@THIRD_PARTY_FEATURE_CONDITION@")
-SET(LIBMYSQL_LOCATION "@LIBMYSQL_LOCATION@")
-
-
 MACRO(MAKE_WIX_IDENTIFIER str varname)
   STRING(REPLACE "/" "." ${varname} "${str}")
   STRING(REGEX REPLACE "[^a-zA-Z_0-9.]" "_" ${varname} "${${varname}}")
@@ -45,19 +13,15 @@ MACRO(MAKE_WIX_IDENTIFIER str varname)
 ENDMACRO()
 
 SET($ENV{VS_UNICODE_OUTPUT} "")
-IF(LIBMYSQL_LOCATION AND CMAKE_CFG_INTDIR) 
- # resolve libmysql full path
- STRING(REPLACE "${CMAKE_CFG_INTDIR}" "${CMAKE_INSTALL_CONFIG_NAME}" LIBMYSQL_LOCATION "${LIBMYSQL_LOCATION}")
-ENDIF()
-	  
+
 FOREACH(third_party ${WITH_THIRD_PARTY})
-  INCLUDE(${CMAKE_CURRENT_SOURCE_DIR}/${third_party}.cmake)
+  INCLUDE(${SRCDIR}/${third_party}.cmake)
  
   # Check than above script produced ${third_party}.wxi and ${third_party}_feature.wxi
   FOREACH(outfile ${third_party}.wxi ${third_party}_feature.wxi)
     IF(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${outfile})
       MESSAGE(FATAL_ERROR 
-       "${CMAKE_CURRENT_SOURCE_DIR}/${third_party}.cmake did not produce "
+       "${SRCDIR}/${third_party}.cmake did not produce "
        "${CMAKE_CURRENT_BINARY_DIR}/${outfile}"
       )
     ENDIF()
@@ -79,7 +43,7 @@ ENDIF()
 
 SET(ENV{VS_UNICODE_OUTPUT})
 
-INCLUDE(${CMAKE_BINARY_DIR}/CPackConfig.cmake)
+INCLUDE(${TOP_BINDIR}/CPackConfig.cmake)
 
 IF(CPACK_WIX_CONFIG)
   INCLUDE(${CPACK_WIX_CONFIG})
@@ -101,13 +65,13 @@ FOREACH(comp ${COMPONENTS_ALL})
  SET(ENV{DESTDIR} testinstall/${comp})
  EXECUTE_PROCESS(
   COMMAND ${CMAKE_COMMAND} ${CONFIG_PARAM} -DCMAKE_INSTALL_COMPONENT=${comp}  
-   -DCMAKE_INSTALL_PREFIX=  -P ${CMAKE_BINARY_DIR}/cmake_install.cmake
+   -DCMAKE_INSTALL_PREFIX=  -P ${TOP_BINDIR}/cmake_install.cmake
    OUTPUT_QUIET
 
   )
   # Exclude empty install components
   SET(INCLUDE_THIS_COMPONENT 1)
-  SET(MANIFEST_FILENAME "${CMAKE_BINARY_DIR}/install_manifest_${comp}.txt")
+  SET(MANIFEST_FILENAME "${TOP_BINDIR}/install_manifest_${comp}.txt")
   IF(EXISTS ${MANIFEST_FILENAME})
     FILE(READ ${MANIFEST_FILENAME} content)
     STRING(LENGTH "${content}" content_length)
@@ -385,9 +349,9 @@ SET(CPACK_WIX_INCLUDES
 ENDFOREACH()
 
 
-CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/mysql_server.wxs.in 
+CONFIGURE_FILE(${SRCDIR}/mysql_server.wxs.in
  ${CMAKE_CURRENT_BINARY_DIR}/mysql_server.wxs)
-CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/extra.wxs.in
+CONFIGURE_FILE(${SRCDIR}/extra.wxs.in
   ${CMAKE_CURRENT_BINARY_DIR}/extra.wxs)
 
 SET(EXTRA_CANDLE_ARGS "$ENV{EXTRA_CANDLE_ARGS}")
@@ -432,6 +396,6 @@ IF(SIGNCODE)
 )
 ENDIF()
 CONFIGURE_FILE(${CPACK_PACKAGE_FILE_NAME}.msi 
- ${CMAKE_BINARY_DIR}/${CPACK_PACKAGE_FILE_NAME}.msi 
+ ${TOP_BINDIR}/${CPACK_PACKAGE_FILE_NAME}.msi
   COPYONLY)
 
diff --git a/win/packaging/extra.wxs.in b/win/packaging/extra.wxs.in
index 819bcc36ecb..a71ef982896 100644
--- a/win/packaging/extra.wxs.in
+++ b/win/packaging/extra.wxs.in
@@ -71,8 +71,8 @@
     <!-- 
       User interface dialogs 
      -->
-    <WixVariable Id='WixUIBannerBmp' Value='@CMAKE_CURRENT_SOURCE_DIR@\WixUIBannerBmp.jpg' />
-    <WixVariable Id='WixUIDialogBmp' Value='@CMAKE_CURRENT_SOURCE_DIR@\WixUIDialogBmp.jpg' />
+    <WixVariable Id='WixUIBannerBmp' Value='@SRCDIR@\WixUIBannerBmp.jpg' />
+    <WixVariable Id='WixUIDialogBmp' Value='@SRCDIR@\WixUIDialogBmp.jpg' />
     <UI>
 
       <!-- Dialog on uninstall of the database -->
diff --git a/win/packaging/heidisql.cmake b/win/packaging/heidisql.cmake
index 569ae4d8ed5..77a0bc05e70 100644
--- a/win/packaging/heidisql.cmake
+++ b/win/packaging/heidisql.cmake
@@ -13,11 +13,5 @@ IF(NOT EXISTS ${HEIDISQL_DOWNLOAD_DIR}/${HEIDISQL_ZIP})
 ENDIF()
 
 SET(LIBMYSQLDLL_SOURCE ${HEIDISQL_DOWNLOAD_DIR}/libmysql.dll)
-IF(CMAKE_SIZEOF_VOID_P EQUAL 4)
-  # Use our libmysql if it is 32 bit.
-  IF(LIBMYSQL_LOCATION)
-    SET(LIBMYSQLDLL_SOURCE "${LIBMYSQL_LOCATION}")
-  ENDIF()
-ENDIF()
 CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/heidisql.wxi.in ${CMAKE_CURRENT_BINARY_DIR}/heidisql.wxi)
 CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/heidisql_feature.wxi.in ${CMAKE_CURRENT_BINARY_DIR}/heidisql_feature.wxi)
diff --git a/win/upgrade_wizard/CMakeLists.txt b/win/upgrade_wizard/CMakeLists.txt
index 81c9c0d572c..f4148ee98d2 100644
--- a/win/upgrade_wizard/CMakeLists.txt
+++ b/win/upgrade_wizard/CMakeLists.txt
@@ -46,9 +46,8 @@ SET_TARGET_PROPERTIES(mysql_upgrade_wizard PROPERTIES WIN32_EXECUTABLE 1)
 # to change service configuration. Due to a CMake bug http://www.vtk.org/Bug/view.php?id=11171
 # it is not possible currenly to do it with linker flags. Work around is to use
 # manifest tool mt.exe and embed the manifest post-build.
-GET_TARGET_PROPERTY(upgrade_wizard_location mysql_upgrade_wizard LOCATION)
 ADD_CUSTOM_COMMAND(
   TARGET mysql_upgrade_wizard POST_BUILD
   COMMAND mt.exe -manifest ${CMAKE_CURRENT_SOURCE_DIR}/upgrade_wizard.exe.manifest
-  "-outputresource:${upgrade_wizard_location};#1"
+  "-outputresource:$<TARGET_FILE:mysql_upgrade_wizard>;#1"
 )

From 9c195176a1e18f4ac7ba1c1bfd91e5fbe2c4a25c Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Tue, 10 Jul 2018 17:44:38 +0200
Subject: [PATCH 140/164] MDEV-16662 CMake warnings: CMP0042

---
 CMakeLists.txt | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0cd4028ac75..3bd46679da4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -19,9 +19,6 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7)
 IF(POLICY CMP0022)
   CMAKE_POLICY(SET CMP0022 NEW)
 ENDIF()
-IF(POLICY CMP0042)
-  CMAKE_POLICY(SET CMP0042 OLD)
-ENDIF()
 IF(POLICY CMP0045)
   CMAKE_POLICY(SET CMP0045 OLD)
 ENDIF()

From 889d8a8f2beb330df27f2b1cf27ea0375dc4444c Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Tue, 10 Jul 2018 17:45:49 +0200
Subject: [PATCH 141/164] MDEV-16662 CMake warnings: CMP0045

---
 CMakeLists.txt | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3bd46679da4..4ceb82cc863 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -19,9 +19,6 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7)
 IF(POLICY CMP0022)
   CMAKE_POLICY(SET CMP0022 NEW)
 ENDIF()
-IF(POLICY CMP0045)
-  CMAKE_POLICY(SET CMP0045 OLD)
-ENDIF()
 IF(POLICY CMP0054)
   CMAKE_POLICY(SET CMP0054 NEW)
 ENDIF()

From 675e7e7dcc89d3b39e96e862c3f69378668584d9 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Wed, 4 Jul 2018 18:21:47 +0200
Subject: [PATCH 142/164] remove obsolete checks for -fno-implicit-templates

we stopped using -fno-implicit-templates many years ago
---
 extra/yassl/CMakeLists.txt                               | 3 ---
 plugin/handler_socket/CMakeLists.txt                     | 9 +--------
 storage/cassandra/CMakeLists.txt                         | 1 -
 storage/mroonga/CMakeLists.txt                           | 1 -
 storage/oqgraph/CMakeLists.txt                           | 1 -
 .../PerconaFT/cmake_modules/TokuSetupCompiler.cmake      | 8 --------
 6 files changed, 1 insertion(+), 22 deletions(-)

diff --git a/extra/yassl/CMakeLists.txt b/extra/yassl/CMakeLists.txt
index c456af9be15..0e28da012ae 100644
--- a/extra/yassl/CMakeLists.txt
+++ b/extra/yassl/CMakeLists.txt
@@ -20,9 +20,6 @@ INCLUDE_DIRECTORIES(
  ${CMAKE_SOURCE_DIR}/extra/yassl/taocrypt/mySTL)
 
 ADD_DEFINITIONS(${SSL_DEFINES})
-#Remove -fno-implicit-templates
-#(yassl sources cannot  be compiled with  it)
-STRING(REPLACE "-fno-implicit-templates" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
 SET(YASSL_SOURCES  src/buffer.cpp src/cert_wrapper.cpp src/crypto_wrapper.cpp src/handshake.cpp src/lock.cpp 
 				src/log.cpp src/socket_wrapper.cpp src/ssl.cpp src/timer.cpp src/yassl_error.cpp 
 				src/yassl_imp.cpp src/yassl_int.cpp)
diff --git a/plugin/handler_socket/CMakeLists.txt b/plugin/handler_socket/CMakeLists.txt
index bd656ebc5b7..9e4707a83fe 100644
--- a/plugin/handler_socket/CMakeLists.txt
+++ b/plugin/handler_socket/CMakeLists.txt
@@ -5,14 +5,7 @@ IF(WIN32)
   RETURN()
 ENDIF()
 
-#Remove -fno-implicit-templates from compiler flags(handlersocket would not work  with  it)
-STRING(REPLACE "-fno-implicit-templates" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
-
-include(CheckCXXCompilerFlag)
-check_cxx_compiler_flag(" -Wdeprecated-declarations" HAVE_CXX_WDEPRECATED_DECLARATIONS)
-IF (HAVE_CXX_WDEPRECATED_DECLARATIONS)
-  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations")
-ENDIF()
+MY_CHECK_AND_SET_COMPILER_FLAG("-Wdeprecated-declarations")
 
 INCLUDE_DIRECTORIES(libhsclient)
 
diff --git a/storage/cassandra/CMakeLists.txt b/storage/cassandra/CMakeLists.txt
index a5d58234d97..7f94c94f69d 100644
--- a/storage/cassandra/CMakeLists.txt
+++ b/storage/cassandra/CMakeLists.txt
@@ -21,7 +21,6 @@ IF(EXISTS ${Thrift_LIBS})
   SET(CMAKE_REQUIRED_INCLUDES ${Thrift_INCLUDE_DIRS})
 
   STRING(REPLACE "-fno-exceptions" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
-  STRING(REPLACE "-fno-implicit-templates" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
 
   SET(CMAKE_REQUIRED_INCLUDES "${Thrift_INCLUDE_DIRS}/..")
   CHECK_CXX_SOURCE_COMPILES(
diff --git a/storage/mroonga/CMakeLists.txt b/storage/mroonga/CMakeLists.txt
index 5a7d4699386..c21eaf70536 100644
--- a/storage/mroonga/CMakeLists.txt
+++ b/storage/mroonga/CMakeLists.txt
@@ -361,7 +361,6 @@ else()
     MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-unused-parameter")
     MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-strict-aliasing")
     MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated")
-    MY_CHECK_AND_SET_COMPILER_FLAG("-fno-implicit-templates")
     if(("${MYSQL_VARIANT}" STREQUAL "MariaDB") OR
         ("${MYSQL_VARIANT}" STREQUAL "MySQL" AND
          ${MYSQL_VERSION} VERSION_LESS "5.7.0"))
diff --git a/storage/oqgraph/CMakeLists.txt b/storage/oqgraph/CMakeLists.txt
index 613a44807c7..638ac00dc01 100644
--- a/storage/oqgraph/CMakeLists.txt
+++ b/storage/oqgraph/CMakeLists.txt
@@ -58,7 +58,6 @@ IF(MSVC)
 ELSE(MSVC)
   # Fix lp bug 1221555 with -fpermissive, so that errors in gcc 4.7 become warnings for the time being
   STRING(REPLACE "-fno-exceptions" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
-  STRING(REPLACE "-fno-implicit-templates" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
   SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated -fno-strict-aliasing -fpermissive")
 ENDIF(MSVC)
 
diff --git a/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake b/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake
index 50d35ee4906..0a5b86a430f 100644
--- a/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake
+++ b/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake
@@ -79,14 +79,6 @@ set_cflags_if_supported(
   )
 ## set_cflags_if_supported_named("-Weffc++" -Weffcpp)
 
-if (CMAKE_CXX_FLAGS MATCHES -fno-implicit-templates)
-  # must append this because mysql sets -fno-implicit-templates and we need to override it
-  check_cxx_compiler_flag(-fimplicit-templates HAVE_CXX_IMPLICIT_TEMPLATES)
-  if (HAVE_CXX_IMPLICIT_TEMPLATES)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fimplicit-templates")
-  endif ()
-endif()
-
 ## Clang has stricter POD checks.  So, only enable this warning on our other builds (Linux + GCC)
 if (NOT CMAKE_CXX_COMPILER_ID MATCHES Clang)
   set_cflags_if_supported(

From 8ca2219065df5a829b9a802bbcc879b64e5cb904 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Wed, 4 Jul 2018 19:50:14 +0200
Subject: [PATCH 143/164] cmake .git warning

only run git if .git exists, don't do it for source tarballs
---
 CMakeLists.txt                 | 2 +-
 libmariadb                     | 2 +-
 storage/rocksdb/CMakeLists.txt | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4ceb82cc863..7779f298ea5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -439,7 +439,7 @@ CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/include/mysql_version.h.in
 CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/sql/sql_builtin.cc.in
     ${CMAKE_BINARY_DIR}/sql/sql_builtin.cc)
 
-IF(GIT_EXECUTABLE)
+IF(GIT_EXECUTABLE AND EXISTS ${PROJECT_SOURCE_DIR}/.git)
   EXECUTE_PROCESS(
     COMMAND ${GIT_EXECUTABLE} rev-parse HEAD
       WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
diff --git a/libmariadb b/libmariadb
index ed5921ed4f2..463211f4a9e 160000
--- a/libmariadb
+++ b/libmariadb
@@ -1 +1 @@
-Subproject commit ed5921ed4f26407ff1ff0596f2c74c8d7098632d
+Subproject commit 463211f4a9e68eb7a2fd7a8a3466e5bf3a13e361
diff --git a/storage/rocksdb/CMakeLists.txt b/storage/rocksdb/CMakeLists.txt
index d37f7e8bf0f..e56c5b1033e 100644
--- a/storage/rocksdb/CMakeLists.txt
+++ b/storage/rocksdb/CMakeLists.txt
@@ -241,7 +241,7 @@ IF (ZSTD_FOUND)
   SET(WITH_ZSTD ON)
 ENDIF()
 
-IF(GIT_EXECUTABLE)
+IF(GIT_EXECUTABLE AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/.git)
   EXECUTE_PROCESS(
     COMMAND ${GIT_EXECUTABLE} rev-parse HEAD
       WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb

From 97b563b5dbfce9dccf5a9e8204272e26d9e414a1 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Thu, 5 Jul 2018 14:23:23 +0200
Subject: [PATCH 144/164] less re-cmake messages

---
 cmake/FindZSTD.cmake                | 18 ------------------
 cmake/Findzstd.cmake                | 25 +++++++++++++++++++++++++
 cmake/package_name.cmake            |  2 +-
 cmake/systemd.cmake                 | 10 ++++++----
 libmariadb                          |  2 +-
 storage/rocksdb/build_rocksdb.cmake |  5 ++---
 6 files changed, 35 insertions(+), 27 deletions(-)
 delete mode 100644 cmake/FindZSTD.cmake
 create mode 100644 cmake/Findzstd.cmake

diff --git a/cmake/FindZSTD.cmake b/cmake/FindZSTD.cmake
deleted file mode 100644
index 0fd73501327..00000000000
--- a/cmake/FindZSTD.cmake
+++ /dev/null
@@ -1,18 +0,0 @@
-find_path(
-    ZSTD_INCLUDE_DIR
-    NAMES "zstd.h"
-)
-
-find_library(
-    ZSTD_LIBRARY
-    NAMES zstd
-)
-
-set(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(
-    ZSTD DEFAULT_MSG ZSTD_INCLUDE_DIR ZSTD_LIBRARIES)
-
-mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARIES ZSTD_FOUND)
-
diff --git a/cmake/Findzstd.cmake b/cmake/Findzstd.cmake
new file mode 100644
index 00000000000..8cd4c248145
--- /dev/null
+++ b/cmake/Findzstd.cmake
@@ -0,0 +1,25 @@
+# - Find zstd
+# Find the zstd compression library and includes
+#
+# ZSTD_INCLUDE_DIR - where to find zstd.h, etc.
+# ZSTD_LIBRARIES - List of libraries when using zstd.
+# ZSTD_FOUND - True if zstd found.
+
+if (DEFINED ZSTD_LIBRARIES)
+  return()
+endif()
+
+find_path(ZSTD_INCLUDE_DIR
+  NAMES zstd.h
+  HINTS ${ZSTD_ROOT_DIR}/include)
+
+find_library(ZSTD_LIBRARIES
+  NAMES zstd
+  HINTS ${ZSTD_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(zstd DEFAULT_MSG ZSTD_LIBRARIES ZSTD_INCLUDE_DIR)
+
+mark_as_advanced(
+  ZSTD_LIBRARIES
+  ZSTD_INCLUDE_DIR)
diff --git a/cmake/package_name.cmake b/cmake/package_name.cmake
index a8bd1596f47..73a53b0766a 100644
--- a/cmake/package_name.cmake
+++ b/cmake/package_name.cmake
@@ -135,7 +135,7 @@ IF(NOT VERSION)
 
   SET(package_name "mariadb${PRODUCT_TAG}-${VERSION}-${SYSTEM_NAME_AND_PROCESSOR}")
 
-  MESSAGE(STATUS "Packaging as: ${package_name}")
+  MESSAGE_ONCE(package_name "Packaging as: ${package_name}")
 
   # Sometimes package suffix is added (something like "-icc-glibc23")
   IF(PACKAGE_SUFFIX)
diff --git a/cmake/systemd.cmake b/cmake/systemd.cmake
index f7365066ef6..fc36d7cefbf 100644
--- a/cmake/systemd.cmake
+++ b/cmake/systemd.cmake
@@ -20,10 +20,12 @@ MACRO(CHECK_SYSTEMD)
     SET(WITH_SYSTEMD "auto" CACHE STRING "Enable systemd scripts and notification support")
     IF(WITH_SYSTEMD STREQUAL "yes" OR WITH_SYSTEMD STREQUAL "auto")
       IF(PKG_CONFIG_FOUND)
-        IF(WITH_SYSTEMD STREQUAL "yes")
-          pkg_search_module(LIBSYSTEMD REQUIRED libsystemd libsystemd-daemon)
-        ELSE()
-          pkg_search_module(LIBSYSTEMD libsystemd libsystemd-daemon)
+        IF (NOT DEFINED LIBSYSTEMD_FOUND)
+          IF(WITH_SYSTEMD STREQUAL "yes")
+            pkg_search_module(LIBSYSTEMD REQUIRED libsystemd libsystemd-daemon)
+          ELSE()
+            pkg_search_module(LIBSYSTEMD libsystemd libsystemd-daemon)
+          ENDIF()
         ENDIF()
         IF(HAVE_DLOPEN)
           SET(LIBSYSTEMD ${LIBSYSTEMD_LIBRARIES})
diff --git a/libmariadb b/libmariadb
index 463211f4a9e..ade0d67b0a7 160000
--- a/libmariadb
+++ b/libmariadb
@@ -1 +1 @@
-Subproject commit 463211f4a9e68eb7a2fd7a8a3466e5bf3a13e361
+Subproject commit ade0d67b0a79cf5e96305b6c312a5b26e9469dd0
diff --git a/storage/rocksdb/build_rocksdb.cmake b/storage/rocksdb/build_rocksdb.cmake
index 0f1b77e255d..8f1a89d3f9f 100644
--- a/storage/rocksdb/build_rocksdb.cmake
+++ b/storage/rocksdb/build_rocksdb.cmake
@@ -33,7 +33,7 @@ endif()
 
 # Optional compression libraries.
 
-foreach(compression_lib LZ4 BZIP2 ZSTD snappy)
+foreach(compression_lib LZ4 BZIP2 zstd snappy)
   FIND_PACKAGE(${compression_lib} QUIET)
 
   SET(WITH_ROCKSDB_${compression_lib} AUTO CACHE STRING
@@ -76,7 +76,7 @@ if(ZSTD_FOUND AND (NOT WITH_ROCKSDB_ZSTD STREQUAL "OFF"))
   if (ZSTD_VALID)
     add_definitions(-DZSTD)
     include_directories(${ZSTD_INCLUDE_DIR})
-    list(APPEND THIRDPARTY_LIBS ${ZSTD_LIBRARY})
+    list(APPEND THIRDPARTY_LIBS ${ZSTD_LIBRARIES})
   endif()
 endif()
 
@@ -392,4 +392,3 @@ target_link_libraries(rocksdblib ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
   set_target_properties(rocksdblib PROPERTIES COMPILE_FLAGS "-fPIC -fno-builtin-memcmp -frtti")
 endif()
-

From 456517b88e665eddd3d197a7b457acd9e1eb75a3 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Thu, 5 Jul 2018 17:37:07 +0200
Subject: [PATCH 145/164] compiler warnings

---
 plugin/handler_socket/CMakeLists.txt | 2 +-
 storage/innobase/fts/fts0opt.cc      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugin/handler_socket/CMakeLists.txt b/plugin/handler_socket/CMakeLists.txt
index 9e4707a83fe..329ff58d7f2 100644
--- a/plugin/handler_socket/CMakeLists.txt
+++ b/plugin/handler_socket/CMakeLists.txt
@@ -5,7 +5,7 @@ IF(WIN32)
   RETURN()
 ENDIF()
 
-MY_CHECK_AND_SET_COMPILER_FLAG("-Wdeprecated-declarations")
+MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-declarations")
 
 INCLUDE_DIRECTORIES(libhsclient)
 
diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc
index 77115049903..c523ac5270e 100644
--- a/storage/innobase/fts/fts0opt.cc
+++ b/storage/innobase/fts/fts0opt.cc
@@ -2657,7 +2657,7 @@ fts_optimize_new_table(
 			empty_slot = i;
 		} else if (slot->table == table) {
 			/* Already exists in our optimize queue. */
-			ut_ad(slot->table_id = table->id);
+			ut_ad(slot->table_id == table->id);
 			return(FALSE);
 		}
 	}

From 655cba619d2e6556d4a7c6a3b9127f248bb8e5a0 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Fri, 10 Aug 2018 15:03:21 +0200
Subject: [PATCH 146/164] Update C/C to v3.0.6

---
 libmariadb                                       | 2 +-
 mysql-test/main/session_tracker_last_gtid.result | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/libmariadb b/libmariadb
index ade0d67b0a7..0e0bf9aa492 160000
--- a/libmariadb
+++ b/libmariadb
@@ -1 +1 @@
-Subproject commit ade0d67b0a79cf5e96305b6c312a5b26e9469dd0
+Subproject commit 0e0bf9aa49278b6c4a52408d4c7c8c7288325d44
diff --git a/mysql-test/main/session_tracker_last_gtid.result b/mysql-test/main/session_tracker_last_gtid.result
index 795d0aaa2a2..deffeca1333 100644
--- a/mysql-test/main/session_tracker_last_gtid.result
+++ b/mysql-test/main/session_tracker_last_gtid.result
@@ -2,7 +2,13 @@
 # MDEV-15477: SESSION_SYSVARS_TRACKER does not track last_gtid
 #
 SET gtid_seq_no=1000;
+-- Tracker : SESSION_TRACK_SCHEMA
+-- test
+
 SET @@session.session_track_system_variables='last_gtid';
+-- Tracker : SESSION_TRACK_SCHEMA
+-- test
+
 create table t1 (a int) engine=innodb;
 -- Tracker : SESSION_TRACK_SYSTEM_VARIABLES
 -- last_gtid

From 5e7496e2eaf01d3be79fca3f199a8d3e597af208 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Sun, 12 Aug 2018 14:52:37 +0200
Subject: [PATCH 147/164] fix C/C packaging on debian

---
 libmariadb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libmariadb b/libmariadb
index 0e0bf9aa492..b948553b346 160000
--- a/libmariadb
+++ b/libmariadb
@@ -1 +1 @@
-Subproject commit 0e0bf9aa49278b6c4a52408d4c7c8c7288325d44
+Subproject commit b948553b3464d37240c4d129fae47ce147e6dabc

From ba10ffe0f4c94c0fd2b1e42615a0c8d1a7f88ab9 Mon Sep 17 00:00:00 2001
From: Sergei Petrunia <psergey@askmonty.org>
Date: Sun, 12 Aug 2018 22:10:32 +0300
Subject: [PATCH 148/164] MDEV-16203: autoinc_debug of rocksdb test suite fails

The test causes simulated server crashes with DBUG_SUICIDE();.

It also relies on transactions that were committed right before the
crash to be visible after the crash (that is, it requires durability).

Run the test with transaction durability enabled: set
rocksdb-flush-log-at-trx-commit=1.
---
 storage/rocksdb/mysql-test/rocksdb/t/autoinc_debug-master.opt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/storage/rocksdb/mysql-test/rocksdb/t/autoinc_debug-master.opt b/storage/rocksdb/mysql-test/rocksdb/t/autoinc_debug-master.opt
index 83ed8522e72..06917181cd6 100644
--- a/storage/rocksdb/mysql-test/rocksdb/t/autoinc_debug-master.opt
+++ b/storage/rocksdb/mysql-test/rocksdb/t/autoinc_debug-master.opt
@@ -1 +1 @@
---binlog-format=row
+--binlog-format=row --rocksdb-flush-log-at-trx-commit=1

From a928751c78a8288e4490b27ef8ae6a39b9708f39 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Mon, 13 Aug 2018 12:14:56 +0200
Subject: [PATCH 149/164] fix bundled pcre unit tests

---
 pcre/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pcre/CMakeLists.txt b/pcre/CMakeLists.txt
index ca60d9ed265..857a96b9ef9 100644
--- a/pcre/CMakeLists.txt
+++ b/pcre/CMakeLists.txt
@@ -377,7 +377,7 @@ MESSAGE(\" \")
 
   IF(UNIX)
     ADD_TEST(NAME pcre_test
-      COMMAND sh pcre_test.sh
+      COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/pcre_test.sh
         ${PROJECT_SOURCE_DIR}
         $<TARGET_FILE:pcretest>
         $<TARGET_FILE_DIR:pcretest>
@@ -387,7 +387,7 @@ MESSAGE(\" \")
   IF(PCRE_BUILD_PCREGREP)
     IF(UNIX)
       ADD_TEST(NAME pcre_grep_test
-        COMMAND sh pcre_grep_test.sh
+        COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/pcre_grep_test.sh
           ${PROJECT_SOURCE_DIR}
           $<TARGET_FILE:pcregrep>
           $<TARGET_FILE:pcretest>

From fac3e575b203e8d6a522a475f9aab4ec5041b146 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Mon, 13 Aug 2018 12:22:55 +0200
Subject: [PATCH 150/164] fix third-party tool packaging on Win

---
 win/packaging/create_msi.cmake | 1 +
 win/packaging/heidisql.cmake   | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/win/packaging/create_msi.cmake b/win/packaging/create_msi.cmake
index f0407684ff3..795baa32c8f 100644
--- a/win/packaging/create_msi.cmake
+++ b/win/packaging/create_msi.cmake
@@ -389,6 +389,7 @@ EXECUTE_PROCESS(
 )
 
 IF(SIGNCODE)
+  SEPARATE_ARGUMENTS(SIGNTOOL_PARAMETERS WINDOWS_COMMAND "${SIGNTOOL_PARAMETERS}")
   EXECUTE_PROCESS(
   COMMAND ${SIGNTOOL_EXECUTABLE} sign ${SIGNTOOL_PARAMETERS} 
   /d ${CPACK_PACKAGE_FILE_NAME}.msi
diff --git a/win/packaging/heidisql.cmake b/win/packaging/heidisql.cmake
index 77a0bc05e70..6f325ca30af 100644
--- a/win/packaging/heidisql.cmake
+++ b/win/packaging/heidisql.cmake
@@ -13,5 +13,5 @@ IF(NOT EXISTS ${HEIDISQL_DOWNLOAD_DIR}/${HEIDISQL_ZIP})
 ENDIF()
 
 SET(LIBMYSQLDLL_SOURCE ${HEIDISQL_DOWNLOAD_DIR}/libmysql.dll)
-CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/heidisql.wxi.in ${CMAKE_CURRENT_BINARY_DIR}/heidisql.wxi)
-CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/heidisql_feature.wxi.in ${CMAKE_CURRENT_BINARY_DIR}/heidisql_feature.wxi)
+CONFIGURE_FILE(${SRCDIR}/heidisql.wxi.in ${CMAKE_CURRENT_BINARY_DIR}/heidisql.wxi)
+CONFIGURE_FILE(${SRCDIR}/heidisql_feature.wxi.in ${CMAKE_CURRENT_BINARY_DIR}/heidisql_feature.wxi)

From 562dd53c290e0ef361976c679bc569a64d098625 Mon Sep 17 00:00:00 2001
From: Sergei Petrunia <psergey@askmonty.org>
Date: Mon, 13 Aug 2018 18:56:52 +0300
Subject: [PATCH 151/164] MDEV-16575: rocksdb.bulk_load_errors fails in
 buildbot with wrong result

Fix a race condition in the test.
---
 storage/rocksdb/mysql-test/rocksdb/t/bulk_load_errors.test | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/storage/rocksdb/mysql-test/rocksdb/t/bulk_load_errors.test b/storage/rocksdb/mysql-test/rocksdb/t/bulk_load_errors.test
index 1e349d0ff18..3f085269365 100644
--- a/storage/rocksdb/mysql-test/rocksdb/t/bulk_load_errors.test
+++ b/storage/rocksdb/mysql-test/rocksdb/t/bulk_load_errors.test
@@ -60,6 +60,13 @@ SELECT * FROM t1;
 
 --source include/wait_until_count_sessions.inc
 
+# Note: in MariaDB, session count will be decremented *before* 
+#   myrocks::rocksdb_close_connection is called. This causes a race condition:
+#   we may grep the error log before bulk load is finalized.
+#   To prevent that, do a soft restart of the server (I wasnt able to find
+#   any other reliable way)
+--source include/restart_mysqld_with_option.inc
+
 --let SEARCH_FILE=$LOG2
 --let SEARCH_PATTERN=RocksDB: Error [0-9]+ finalizing last SST file while disconnecting
 --source include/search_pattern_in_file.inc

From 1faaaa9718e1c5b09dc5822f48f3a48a64d784cb Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Mon, 13 Aug 2018 22:39:31 +0100
Subject: [PATCH 152/164] MDEV-15680 xb_aws_key_management fails in buildbot.

aws_key_management needs current directory to be datadir during
initalization, it scans current directory for encrypted keys.

Fix is to ensure, that plugin initialization in mariabackup happens
after the call to my_setwd(mysql_real_data_home).
---
 extra/mariabackup/xtrabackup.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc
index 9c961af2583..4545e7a6857 100644
--- a/extra/mariabackup/xtrabackup.cc
+++ b/extra/mariabackup/xtrabackup.cc
@@ -3804,7 +3804,7 @@ xtrabackup_backup_func()
 		return(false);
 	}
 	msg("mariabackup: cd to %s\n", mysql_real_data_home);
-
+	encryption_plugin_backup_init(mysql_connection);
 	msg("mariabackup: open files limit requested %u, set to %u\n",
 	    (uint) xb_open_files_limit,
 	    xb_set_max_open_files(xb_open_files_limit));
@@ -5277,7 +5277,6 @@ xb_init()
 			return(false);
 		}
 
-		encryption_plugin_backup_init(mysql_connection);
 		history_start_time = time(NULL);
 
 	}

From 9a4998a35a0c402fa2dd6d53fbef3ac9897f9e0a Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Mon, 13 Aug 2018 23:42:20 +0100
Subject: [PATCH 153/164] adjust result file

---
 mysql-test/suite/mariabackup/xb_aws_key_management.result | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mysql-test/suite/mariabackup/xb_aws_key_management.result b/mysql-test/suite/mariabackup/xb_aws_key_management.result
index fd12344dfa0..6efc76a8be3 100644
--- a/mysql-test/suite/mariabackup/xb_aws_key_management.result
+++ b/mysql-test/suite/mariabackup/xb_aws_key_management.result
@@ -11,4 +11,6 @@ INSERT INTO t VALUES('foobar1');
 SELECT * from t;
 c
 foobar1
+Warnings:
+Note	1105	AWS KMS plugin: loaded key 1, version 1, key length 128 bit
 DROP TABLE t;

From 922e7badfcb3f0ab60d1b53e6ab3a771edd6ee0a Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Thu, 9 Aug 2018 15:06:52 +0100
Subject: [PATCH 154/164] MDEV-16791  mariabackup : Support DDL commands during
 backup

---
 extra/mariabackup/backup_copy.cc              |  30 +
 extra/mariabackup/backup_mysql.cc             |   7 +-
 extra/mariabackup/datasink.c                  |   3 +
 extra/mariabackup/fil_cur.cc                  |  38 +-
 extra/mariabackup/fil_cur.h                   |   5 +-
 extra/mariabackup/xtrabackup.cc               | 557 ++++++++++++++++--
 .../mariabackup/create_during_backup.result   |  10 +
 .../mariabackup/create_during_backup.test     |  26 +
 ...e_with_data_directory_during_backup.result |  10 +
 ...ate_with_data_directory_during_backup.test |  24 +
 mysql-test/suite/mariabackup/disabled.def     |   2 +-
 .../drop_table_during_backup.result           |  13 +
 .../mariabackup/drop_table_during_backup.test |  24 +
 .../incremental_ddl_during_backup.result      |  22 +
 .../incremental_ddl_during_backup.test        |  52 ++
 .../suite/mariabackup/mlog_index_load.result  |  15 +
 .../suite/mariabackup/mlog_index_load.test    |  27 +
 .../recreate_table_during_backup.result       |  23 +
 .../recreate_table_during_backup.test         |  37 ++
 .../mariabackup/rename_during_backup.result   |  53 ++
 .../mariabackup/rename_during_backup.test     |  86 +++
 .../mariabackup/rename_during_mdl_lock.result |  10 +-
 .../mariabackup/rename_during_mdl_lock.test   |  20 +-
 mysql-test/suite/mariabackup/suite.opt        |   2 +-
 storage/innobase/include/log0recv.h           |  20 +-
 storage/innobase/log/log0recv.cc              |  38 +-
 26 files changed, 1074 insertions(+), 80 deletions(-)
 create mode 100644 mysql-test/suite/mariabackup/create_during_backup.result
 create mode 100644 mysql-test/suite/mariabackup/create_during_backup.test
 create mode 100644 mysql-test/suite/mariabackup/create_with_data_directory_during_backup.result
 create mode 100644 mysql-test/suite/mariabackup/create_with_data_directory_during_backup.test
 create mode 100644 mysql-test/suite/mariabackup/drop_table_during_backup.result
 create mode 100644 mysql-test/suite/mariabackup/drop_table_during_backup.test
 create mode 100644 mysql-test/suite/mariabackup/incremental_ddl_during_backup.result
 create mode 100644 mysql-test/suite/mariabackup/incremental_ddl_during_backup.test
 create mode 100644 mysql-test/suite/mariabackup/mlog_index_load.result
 create mode 100644 mysql-test/suite/mariabackup/mlog_index_load.test
 create mode 100644 mysql-test/suite/mariabackup/recreate_table_during_backup.result
 create mode 100644 mysql-test/suite/mariabackup/recreate_table_during_backup.test
 create mode 100644 mysql-test/suite/mariabackup/rename_during_backup.result
 create mode 100644 mysql-test/suite/mariabackup/rename_during_backup.test

diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc
index 30e94153275..3a5a5bc5a2a 100644
--- a/extra/mariabackup/backup_copy.cc
+++ b/extra/mariabackup/backup_copy.cc
@@ -1388,6 +1388,30 @@ out:
 	return(ret);
 }
 
+void backup_fix_ddl(void);
+
+#define LSN_PREFIX_IN_SHOW_STATUS  "\nLog sequence number "
+static lsn_t get_current_lsn(MYSQL *connection) {
+	MYSQL_RES *res = xb_mysql_query(connection, "SHOW ENGINE INNODB STATUS", true, false);
+	if (!res)
+		return 0;
+	MYSQL_ROW row = mysql_fetch_row(res);
+	DBUG_ASSERT(row);
+	if (row) {
+		const char *p = strstr(row[2],LSN_PREFIX_IN_SHOW_STATUS);
+		DBUG_ASSERT(p);
+		if (p)
+		{
+			p += sizeof(LSN_PREFIX_IN_SHOW_STATUS) - 1;
+			return (lsn_t)strtoll(p, NULL, 10);
+		}
+	}
+	mysql_free_result(res);
+	return 0;
+}
+
+lsn_t server_lsn_after_lock;
+extern void backup_wait_for_lsn(lsn_t lsn);
 /** Start --backup */
 bool backup_start()
 {
@@ -1407,6 +1431,7 @@ bool backup_start()
 		if (!lock_tables(mysql_connection)) {
 			return(false);
 		}
+		server_lsn_after_lock = get_current_lsn(mysql_connection);
 	}
 
 	if (!backup_files(fil_path_to_mysql_datadir, false)) {
@@ -1421,6 +1446,10 @@ bool backup_start()
 		rocksdb_create_checkpoint();
 	}
 
+	msg_ts("Waiting for log copy thread to read lsn %llu\n", (ulonglong)server_lsn_after_lock);
+	backup_wait_for_lsn(server_lsn_after_lock);
+	backup_fix_ddl();
+
 	// There is no need to stop slave thread before coping non-Innodb data when
 	// --no-lock option is used because --no-lock option requires that no DDL or
 	// DML to non-transaction tables can occur.
@@ -2230,6 +2259,7 @@ static void rocksdb_lock_checkpoint()
 		msg_ts("Could not obtain rocksdb checkpont lock\n");
 		exit(EXIT_FAILURE);
 	}
+	mysql_free_result(res);
 }
 
 static void rocksdb_unlock_checkpoint()
diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc
index 60dfb66d2c8..efa21965965 100644
--- a/extra/mariabackup/backup_mysql.cc
+++ b/extra/mariabackup/backup_mysql.cc
@@ -1794,7 +1794,12 @@ mdl_lock_table(ulint space_id)
     std::ostringstream lock_query;
     lock_query << "SELECT 1 FROM " << full_table_name  << " LIMIT 0";
     msg_ts("Locking MDL for %s\n", full_table_name.c_str());
-    xb_mysql_query(mdl_con, lock_query.str().c_str(), false, true);
+    if (mysql_query(mdl_con, lock_query.str().c_str())) {
+      msg_ts("Warning : locking MDL failed for space id %zu, name %s\n", space_id, full_table_name.c_str());
+    } else {
+      MYSQL_RES *r = mysql_store_result(mdl_con);
+      mysql_free_result(r);
+    }
   }
 
   pthread_mutex_unlock(&mdl_lock_con_mutex);
diff --git a/extra/mariabackup/datasink.c b/extra/mariabackup/datasink.c
index 1459da2fb57..cb31538726f 100644
--- a/extra/mariabackup/datasink.c
+++ b/extra/mariabackup/datasink.c
@@ -108,6 +108,9 @@ Write to a datasink file.
 int
 ds_write(ds_file_t *file, const void *buf, size_t len)
 {
+	if (len == 0) {
+		return 0;
+	}
 	return file->datasink->write(file, (const uchar *)buf, len);
 }
 
diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc
index f6e6fe43085..be2e0593129 100644
--- a/extra/mariabackup/fil_cur.cc
+++ b/extra/mariabackup/fil_cur.cc
@@ -131,14 +131,15 @@ Open a source file cursor and initialize the associated read filter.
 be skipped and XB_FIL_CUR_ERROR on error. */
 xb_fil_cur_result_t
 xb_fil_cur_open(
-/*============*/
+	/*============*/
 	xb_fil_cur_t*	cursor,		/*!< out: source file cursor */
 	xb_read_filt_t*	read_filter,	/*!< in/out: the read filter */
 	fil_node_t*	node,		/*!< in: source tablespace node */
-	uint		thread_n)	/*!< thread number for diagnostics */
+	uint		thread_n,	/*!< thread number for diagnostics */
+	ulonglong max_file_size)
 {
 	bool	success;
-
+	int err;
 	/* Initialize these first so xb_fil_cur_close() handles them correctly
 	in case of error */
 	cursor->orig_buf = NULL;
@@ -173,7 +174,7 @@ xb_fil_cur_open(
 			    "tablespace %s\n",
 			    thread_n, cursor->abs_path);
 
-			return(XB_FIL_CUR_ERROR);
+			return(XB_FIL_CUR_SKIP);
 		}
 		mutex_enter(&fil_system->mutex);
 
@@ -194,14 +195,31 @@ xb_fil_cur_open(
 
 	cursor->node = node;
 	cursor->file = node->handle;
-
-	if (stat(cursor->abs_path, &cursor->statinfo)) {
-		msg("[%02u] mariabackup: error: cannot stat %s\n",
+#ifdef _WIN32
+    HANDLE hDup;
+    DuplicateHandle(GetCurrentProcess(),cursor->file.m_file,
+		GetCurrentProcess(), &hDup, 0, FALSE, DUPLICATE_SAME_ACCESS);
+	int filenr = _open_osfhandle((intptr_t)hDup, 0);
+	if (filenr < 0) {
+		err = EINVAL;
+	}
+	else {
+		err = _fstat64(filenr, &cursor->statinfo);
+		close(filenr);
+	}
+#else
+	err = fstat(cursor->file.m_file, &cursor->statinfo);
+#endif
+	if (max_file_size < (ulonglong)cursor->statinfo.st_size) {
+		cursor->statinfo.st_size = (ulonglong)max_file_size;
+	}
+	if (err) {
+		msg("[%02u] mariabackup: error: cannot fstat %s\n",
 		    thread_n, cursor->abs_path);
 
 		xb_fil_cur_close(cursor);
 
-		return(XB_FIL_CUR_ERROR);
+		return(XB_FIL_CUR_SKIP);
 	}
 
 	if (srv_file_flush_method == SRV_O_DIRECT
@@ -374,7 +392,9 @@ xb_fil_cur_close(
 /*=============*/
 	xb_fil_cur_t *cursor)	/*!< in/out: source file cursor */
 {
-	cursor->read_filter->deinit(&cursor->read_filter_ctxt);
+	if (cursor->read_filter) {
+		cursor->read_filter->deinit(&cursor->read_filter_ctxt);
+	}
 
 	free(cursor->orig_buf);
 
diff --git a/extra/mariabackup/fil_cur.h b/extra/mariabackup/fil_cur.h
index e3f356a346c..4d7f7fd84d9 100644
--- a/extra/mariabackup/fil_cur.h
+++ b/extra/mariabackup/fil_cur.h
@@ -57,7 +57,7 @@ struct xb_fil_cur_t {
 	ulint		space_size;	/*!< space size in pages */
 
 	/** TODO: remove this default constructor */
-	xb_fil_cur_t() : page_size(0), read_filter_ctxt() {}
+	xb_fil_cur_t() : page_size(0), read_filter(0), read_filter_ctxt() {}
 
 	/** @return whether this is not a file-per-table tablespace */
 	bool is_system() const
@@ -86,7 +86,8 @@ xb_fil_cur_open(
 	xb_fil_cur_t*	cursor,		/*!< out: source file cursor */
 	xb_read_filt_t*	read_filter,	/*!< in/out: the read filter */
 	fil_node_t*	node,		/*!< in: source tablespace node */
-	uint		thread_n);	/*!< thread number for diagnostics */
+	uint		thread_n,	/*!< thread number for diagnostics */
+	ulonglong max_file_size = ULLONG_MAX);
 
 /************************************************************************
 Reads and verifies the next block of pages from the source
diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc
index 4545e7a6857..a7d136be820 100644
--- a/extra/mariabackup/xtrabackup.cc
+++ b/extra/mariabackup/xtrabackup.cc
@@ -343,6 +343,25 @@ const char *opt_history = NULL;
 char mariabackup_exe[FN_REFLEN];
 char orig_argv1[FN_REFLEN];
 
+pthread_mutex_t backup_mutex;
+pthread_cond_t  scanned_lsn_cond;
+
+typedef std::map<space_id_t,std::string> space_id_to_name_t;
+
+struct ddl_tracker_t {
+	/** Tablspaces with their ID and name, as they were copied to backup.*/
+	space_id_to_name_t tables_in_backup;
+	/** Tablespaces for that optimized DDL without redo log was found.*/
+	std::set<space_id_t> optimized_ddl;
+	/** Drop operations found in redo log. */
+	std::set<space_id_t> drops;
+	/* For DDL operation found in redo log,  */
+	space_id_to_name_t id_to_name;
+};
+const space_id_t REMOVED_SPACE_ID = ULINT_MAX;
+static ddl_tracker_t ddl_tracker;
+
+
 /* Whether xtrabackup_binlog_info should be created on recovery */
 static bool recover_binlog_info;
 
@@ -537,49 +556,79 @@ void mdl_lock_all()
 		mdl_lock_table(node->space->id);
 	}
 	datafiles_iter_free(it);
-
-	DBUG_EXECUTE_IF("check_mdl_lock_works",
-		dbug_alter_thread_done =
-		  dbug_start_query_thread("ALTER TABLE test.t ADD COLUMN mdl_lock_column int",
-			 "Waiting for table metadata lock",1, ER_QUERY_INTERRUPTED););
 }
 
-/** Check if the space id belongs to the table which name should
-be skipped based on the --tables, --tables-file and --table-exclude
-options.
-@param[in]	space_id	space id to check
-@return true if the space id belongs to skip table/database list. */
-static bool backup_includes(space_id_t space_id)
+
+// Convert non-null terminated filename to space name
+std::string filename_to_spacename(const byte *filename, size_t len)
 {
-	datafiles_iter_t *it = datafiles_iter_new(fil_system);
-	if (!it)
-		return true;
+	// null- terminate filename
+	char *f = (char *)malloc(len + 1);
+	ut_a(f);
+	memcpy(f, filename, len);
+	f[len] = 0;
+	for (size_t i = 0; i < len; i++)
+		if (f[i] == '\\')
+			f[i] = '/';
+	char *p = strrchr(f, '.');
+	ut_a(p);
+	*p = 0;
+	char *table = strrchr(f, '/');
+	ut_a(table);
+	*table = 0;
+	char *db = strrchr(f, '/');
+	ut_a(db);
+	*table = '/';
+	return std::string(db+1);
+}
 
-	while (fil_node_t *node = datafiles_iter_next(it)){
-		if (space_id == 0
-		    || (node->space->id == space_id
-			&& !check_if_skip_table(node->space->name))) {
+/** Report an operation to create, delete, or rename a file during backup.
+@param[in]	space_id	tablespace identifier
+@param[in]	flags		tablespace flags (NULL if not create)
+@param[in]	name		file name (not NUL-terminated)
+@param[in]	len		length of name, in bytes
+@param[in]	new_name	new file name (NULL if not rename)
+@param[in]	new_len		length of new_name, in bytes (0 if NULL) */
+void backup_file_op(ulint space_id, const byte* flags,
+	const byte* name, ulint len,
+	const byte* new_name, ulint new_len)
+{
 
-			msg("mariabackup: Unsupported redo log detected "
-			"and it belongs to %s\n",
-			space_id ? node->name: "the InnoDB system tablespace");
+	ut_ad(!flags || !new_name);
+	ut_ad(name);
+	ut_ad(len);
+	ut_ad(!new_name == !new_len);
+	pthread_mutex_lock(&backup_mutex);
 
-			msg("mariabackup: ALTER TABLE or OPTIMIZE TABLE "
-			"was being executed during the backup.\n");
-
-			if (!opt_lock_ddl_per_table) {
-				msg("mariabackup: Use --lock-ddl-per-table "
-				"parameter to lock all the table before "
-				"backup operation.\n");
-			}
-
-			datafiles_iter_free(it);
-			return false;
-		}
+	if (flags) {
+		ddl_tracker.id_to_name[space_id] = filename_to_spacename(name, len);
+		msg("DDL tracking :  create %zu \"%.*s\": %x\n",
+			space_id, int(len), name, mach_read_from_4(flags));
 	}
+	else if (new_name) {
+		ddl_tracker.id_to_name[space_id] = filename_to_spacename(new_name, new_len);
+		msg("DDL tracking : rename %zu \"%.*s\",\"%.*s\"\n",
+			space_id, int(len), name, int(new_len), new_name);
+	} else {
+		ddl_tracker.drops.insert(space_id);
+		msg("DDL tracking : delete %zu \"%.*s\"\n", space_id, int(len), name);
+	}
+	pthread_mutex_unlock(&backup_mutex);
+}
 
-	datafiles_iter_free(it);
-	return true;
+
+/** Callback whenever MLOG_INDEX_LOAD happens.
+@param[in]	space_id	space id to check
+@return false */
+void backup_optimized_ddl_op(ulint space_id)
+{
+	// TODO : handle incremental
+	if (xtrabackup_incremental)
+		return;
+
+	pthread_mutex_lock(&backup_mutex);
+	ddl_tracker.optimized_ddl.insert(space_id);
+	pthread_mutex_unlock(&backup_mutex);
 }
 
 /* ======== Date copying thread context ======== */
@@ -2336,7 +2385,7 @@ xb_get_copy_action(const char *dflt)
 
 static
 my_bool
-xtrabackup_copy_datafile(fil_node_t* node, uint thread_n)
+xtrabackup_copy_datafile(fil_node_t* node, uint thread_n, const char *dest_name=0, ulonglong max_size=ULLONG_MAX)
 {
 	char			 dst_name[FN_REFLEN];
 	ds_file_t		*dstfile = NULL;
@@ -2366,20 +2415,31 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n)
 		return(FALSE);
 	}
 
+	bool was_dropped;
+	pthread_mutex_lock(&backup_mutex);
+	was_dropped = (ddl_tracker.drops.find(node->space->id) != ddl_tracker.drops.end());
+	pthread_mutex_unlock(&backup_mutex);
+	if (was_dropped) {
+		fil_space_close(node->space->name);
+		goto skip;
+	}
+
 	if (!changed_page_bitmap) {
 		read_filter = &rf_pass_through;
 	}
 	else {
 		read_filter = &rf_bitmap;
 	}
-	res = xb_fil_cur_open(&cursor, read_filter, node, thread_n);
+
+	res = xb_fil_cur_open(&cursor, read_filter, node, thread_n,max_size);
 	if (res == XB_FIL_CUR_SKIP) {
 		goto skip;
 	} else if (res == XB_FIL_CUR_ERROR) {
 		goto error;
 	}
 
-	strncpy(dst_name, cursor.rel_path, sizeof(dst_name));
+	strncpy(dst_name, (dest_name)?dest_name : cursor.rel_path, sizeof(dst_name));
+
 
 	/* Setup the page write filter */
 	if (xtrabackup_incremental) {
@@ -2431,6 +2491,10 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n)
 		goto error;
 	}
 
+	pthread_mutex_lock(&backup_mutex);
+	ddl_tracker.tables_in_backup[node->space->id] = node_name;
+	pthread_mutex_unlock(&backup_mutex);
+
 	/* close */
 	msg_ts("[%02u]        ...done\n", thread_n);
 	xb_fil_cur_close(&cursor);
@@ -2595,7 +2659,7 @@ static bool xtrabackup_copy_logfile(bool last = false)
 		if (!start_lsn) {
 			msg("mariabackup: Error: xtrabackup_copy_logfile()"
 			    " failed.\n");
-			return(true);
+			exit(EXIT_FAILURE);
 		}
 	} while (start_lsn == end_lsn);
 
@@ -2604,12 +2668,30 @@ static bool xtrabackup_copy_logfile(bool last = false)
 	msg_ts(">> log scanned up to (" LSN_PF ")\n", start_lsn);
 
 	/* update global variable*/
+	pthread_mutex_lock(&backup_mutex);
 	log_copy_scanned_lsn = start_lsn;
+	pthread_cond_broadcast(&scanned_lsn_cond);
+	pthread_mutex_unlock(&backup_mutex);
 
 	debug_sync_point("xtrabackup_copy_logfile_pause");
 	return(false);
 }
 
+/**
+Wait until redo log copying thread processes given lsn
+*/
+void backup_wait_for_lsn(lsn_t lsn) {
+	bool completed = false;
+	pthread_mutex_lock(&backup_mutex);
+	do {
+		pthread_cond_wait(&scanned_lsn_cond, &backup_mutex);
+		completed = log_copy_scanned_lsn >= lsn;
+	} while (!completed);
+	pthread_mutex_unlock(&backup_mutex);
+}
+
+extern lsn_t server_lsn_after_lock;
+
 static os_thread_ret_t log_copying_thread(void*)
 {
 	/*
@@ -2666,6 +2748,42 @@ static os_thread_ret_t io_watching_thread(void*)
 	return(0);
 }
 
+#ifndef DBUG_OFF
+/*
+In debug mode,  execute SQL statement that was passed via environment.
+To use this facility, you need to
+
+1. Add code DBUG_EXECUTE_MARIABACKUP_EVENT("my_event_name", key););
+  to the code. key is usually a table name
+2. Set environment variable my_event_name_$key SQL statement you want to execute
+   when event occurs, in DBUG_EXECUTE_IF from above.
+   In mtr , you can set environment via 'let' statement (do not use $ as the first char
+   for the variable)
+3. start mariabackup with --dbug=+d,debug_mariabackup_events
+*/
+static void dbug_mariabackup_event(const char *event,const char *key)
+{
+	char envvar[FN_REFLEN];
+	if (key) {
+		snprintf(envvar, sizeof(envvar), "%s_%s", event, key);
+		char *slash = strchr(envvar, '/');
+		if (slash)
+			*slash = '_';
+	} else {
+		strncpy(envvar, event, sizeof(envvar));
+	}
+	char *sql = getenv(envvar);
+	if (sql) {
+		msg("dbug_mariabackup_event : executing '%s'\n", sql);
+		xb_mysql_query(mysql_connection, sql, false, true);
+	}
+
+}
+#define DBUG_MARIABACKUP_EVENT(A, B) DBUG_EXECUTE_IF("mariabackup_events", dbug_mariabackup_event(A,B););
+#else
+#define DBUG_MARIABACKUP_EVENT(A,B)
+#endif
+
 /**************************************************************************
 Datafiles copying thread.*/
 static
@@ -2688,12 +2806,18 @@ data_copy_thread_func(
 
 	while ((node = datafiles_iter_next(ctxt->it)) != NULL) {
 
+		DBUG_MARIABACKUP_EVENT("before_copy", node->space->name);
+
+
 		/* copy the datafile */
 		if(xtrabackup_copy_datafile(node, num)) {
 			msg("[%02u] mariabackup: Error: "
 			    "failed to copy datafile.\n", num);
 			exit(EXIT_FAILURE);
 		}
+
+		DBUG_MARIABACKUP_EVENT("after_copy", node->space->name);
+
 	}
 
 	pthread_mutex_lock(&ctxt->count_mutex);
@@ -2866,6 +2990,7 @@ xb_load_single_table_tablespace(
 	Datafile *file = xb_new_datafile(name, is_remote);
 
 	if (file->open_read_only(true) != DB_SUCCESS) {
+		msg("Can't open datafile %s\n", name);
 		ut_free(name);
 		exit(EXIT_FAILURE);
 	}
@@ -3182,7 +3307,7 @@ xb_load_tablespaces()
 	}
 
 	debug_sync_point("xtrabackup_load_tablespaces_pause");
-
+	DBUG_MARIABACKUP_EVENT("after_load_tablespaces", 0);
 	return(DB_SUCCESS);
 }
 
@@ -3791,6 +3916,8 @@ xtrabackup_backup_func()
 	uint			 count;
 	pthread_mutex_t		 count_mutex;
 	data_thread_ctxt_t 	*data_threads;
+	pthread_mutex_init(&backup_mutex, NULL);
+	pthread_cond_init(&scanned_lsn_cond, NULL);
 
 #ifdef USE_POSIX_FADVISE
 	msg("mariabackup: uses posix_fadvise().\n");
@@ -3817,6 +3944,7 @@ xtrabackup_backup_func()
 	srv_read_only_mode = TRUE;
 
 	srv_operation = SRV_OPERATION_BACKUP;
+	log_file_op = backup_file_op;
 	metadata_to_lsn = 0;
 
 	if (xb_close_files)
@@ -3830,6 +3958,7 @@ xtrabackup_backup_func()
 fail:
 		metadata_to_lsn = log_copying_running;
 		stop_backup_threads();
+		log_file_op = NULL;
 		if (dst_log_file) {
 			ds_close(dst_log_file);
 			dst_log_file = NULL;
@@ -4116,6 +4245,7 @@ fail_before_log_copying_thread_start:
 		goto fail_before_log_copying_thread_start;
 
 	log_copying_stop = os_event_create(0);
+	log_optimized_ddl_op = backup_optimized_ddl_op;
 	os_thread_create(log_copying_thread, NULL, &log_copying_thread_id);
 
 	/* FLUSH CHANGED_PAGE_BITMAPS call */
@@ -4147,6 +4277,11 @@ fail_before_log_copying_thread_start:
 
 	if (opt_lock_ddl_per_table) {
 		mdl_lock_all();
+
+		DBUG_EXECUTE_IF("check_mdl_lock_works",
+			dbug_alter_thread_done =
+			dbug_start_query_thread("ALTER TABLE test.t ADD COLUMN mdl_lock_column int",
+				"Waiting for table metadata lock", 1, ER_QUERY_INTERRUPTED););
 	}
 
 	it = datafiles_iter_new(fil_system);
@@ -4184,10 +4319,6 @@ fail_before_log_copying_thread_start:
 	pthread_mutex_destroy(&count_mutex);
 	free(data_threads);
 	datafiles_iter_free(it);
-
-	if (changed_page_bitmap) {
-		xb_page_bitmap_deinit(changed_page_bitmap);
-	}
 	}
 
 	bool ok = backup_start();
@@ -4211,6 +4342,9 @@ fail_before_log_copying_thread_start:
 		goto fail;
 	}
 
+	if (changed_page_bitmap) {
+		xb_page_bitmap_deinit(changed_page_bitmap);
+	}
 	xtrabackup_destroy_datasinks();
 
 	msg("mariabackup: Redo log (from LSN " LSN_PF " to " LSN_PF
@@ -4228,9 +4362,178 @@ fail_before_log_copying_thread_start:
 	}
 
 	innodb_shutdown();
+	log_file_op = NULL;
+	pthread_mutex_destroy(&backup_mutex);
+	pthread_cond_destroy(&scanned_lsn_cond);
 	return(true);
 }
 
+
+/**
+This function handles DDL changes at the end of backup, under protection of
+FTWRL.  This ensures consistent backup in presence of DDL.
+
+- New tables, that were created during backup, are now copied into backup.
+  Also, tablespaces with optimized (no redo loggin DDL) are re-copied into 
+  backup. This tablespaces will get the extension ".new" in the backup
+
+- Tables that were renamed during backup, are marked as renamed
+  For these, file <old_name>.ren will be created.
+  The content of the file is the new tablespace name.
+
+- Tables that were deleted during backup, are marked as deleted
+  For these , an empty file <name>.del will be created
+
+  It is the responsibility of the prepare phase to deal with .new, .ren, and .del
+  files.
+*/
+void backup_fix_ddl(void)
+{
+	std::set<std::string> new_tables;
+	std::set<std::string> dropped_tables;
+	std::map<std::string, std::string> renamed_tables;
+
+	for (space_id_to_name_t::iterator iter = ddl_tracker.tables_in_backup.begin();
+		iter != ddl_tracker.tables_in_backup.end();
+		iter++) {
+
+		const std::string name = iter->second;
+		ulint id = iter->first;
+
+		if (ddl_tracker.drops.find(id) != ddl_tracker.drops.end()) {
+			dropped_tables.insert(name);
+			continue;
+		}
+
+		bool has_optimized_ddl =
+			ddl_tracker.optimized_ddl.find(id) != ddl_tracker.optimized_ddl.end();
+
+		if (ddl_tracker.id_to_name.find(id) == ddl_tracker.id_to_name.end()) {
+			if (has_optimized_ddl) {
+				new_tables.insert(name);
+			}
+			continue;
+		}
+
+		/* tablespace was affected by DDL. */
+		const std::string new_name = ddl_tracker.id_to_name[id];
+		if (new_name != name) {
+			if (has_optimized_ddl) {
+				/* table was renamed, but we need a full copy
+				of it because of optimized DDL. We emulate a drop/create.*/
+				dropped_tables.insert(name);
+				new_tables.insert(new_name);
+			} else {
+				/* Renamed, and no optimized DDL*/
+				renamed_tables[name] = new_name;
+			}
+		} else if (has_optimized_ddl) {
+			/* Table was recreated, or optimized DDL ran.
+			In both cases we need a full copy in the backup.*/
+			new_tables.insert(name);
+		}
+	}
+
+	/* Find tables that were created during backup (and not removed).*/
+	for(space_id_to_name_t::iterator iter = ddl_tracker.id_to_name.begin();
+		iter != ddl_tracker.id_to_name.end();
+		iter++) {
+
+		ulint id = iter->first;
+		std::string name = iter->second;
+
+		if (ddl_tracker.tables_in_backup.find(id) != ddl_tracker.tables_in_backup.end()) {
+			/* already processed above */
+			continue;
+		}
+
+		if (ddl_tracker.drops.find(id) == ddl_tracker.drops.end()) {
+			dropped_tables.erase(name);
+			new_tables.insert(name);
+		}
+	}
+
+	// Mark tablespaces for rename
+	for (std::map<std::string, std::string>::iterator iter = renamed_tables.begin();
+		iter != renamed_tables.end(); ++iter) {
+		const std::string old_name = iter->first;
+		std::string new_name = iter->second;
+		backup_file_printf((old_name + ".ren").c_str(), "%s", new_name.c_str());
+	}
+
+	// Mark tablespaces for drop
+	for (std::set<std::string>::iterator iter = dropped_tables.begin();
+		iter != dropped_tables.end();
+		iter++) {
+		const std::string name(*iter);
+		backup_file_printf((name + ".del").c_str(), "%s", "");
+	}
+
+	//  Load and copy new tables.
+	//  Close all datanodes first, reload only new tables.
+	std::vector<fil_node_t *> all_nodes;
+	datafiles_iter_t *it = datafiles_iter_new(fil_system);
+	if (!it)
+		return;
+	while (fil_node_t *node = datafiles_iter_next(it)) {
+		all_nodes.push_back(node);
+	}
+	for (size_t i = 0; i < all_nodes.size(); i++) {
+		fil_node_t *n = all_nodes[i];
+		if (n->space->id == 0)
+			continue;
+		fil_space_close(n->space->name);
+		fil_space_free(n->space->id, false);
+	}
+
+
+	for (std::set<std::string>::iterator iter = new_tables.begin();
+		iter != new_tables.end(); iter++) {
+		const char *space_name = iter->c_str();
+		if (check_if_skip_table(space_name))
+			continue;
+		std::string name(*iter);
+		bool is_remote = access((name + ".ibd").c_str(), R_OK) != 0;
+		const char *extension = is_remote ? ".isl" : ".ibd";
+		name.append(extension);
+		char buf[FN_REFLEN];
+		strncpy(buf, name.c_str(), sizeof(buf));
+		const char *dbname = buf;
+		char *p = strchr(buf, '/');
+		if (p == 0) {
+			msg("Unexpected tablespace %s filename %s\n", space_name, name.c_str());
+			ut_a(0);
+		}
+		ut_a(p);
+		*p = 0;
+		const char *tablename = p + 1;
+		xb_load_single_table_tablespace(dbname, tablename, is_remote);
+	}
+
+	it = datafiles_iter_new(fil_system);
+	if (!it)
+		return;
+
+	while (fil_node_t *node = datafiles_iter_next(it)) {
+		fil_space_t * space = node->space;
+		if (!fil_is_user_tablespace_id(space->id))
+			continue;
+		std::string dest_name(node->space->name);
+		dest_name.append(".new");
+#if 0
+		bool do_full_copy = ddl_tracker.optimized_ddl.find(n->space->id) != ddl_tracker.optimized_ddl.end();
+		if (do_full_copy) {
+			msg(
+				"Performing a full copy of the tablespace %s, because optimized (without redo logging) DDL operation"
+				"ran during backup. You can use set innodb_log_optimize_ddl=OFF to improve backup performance"
+				"in the future.\n",
+				n->space->name);
+		}
+#endif
+		xtrabackup_copy_datafile(node, 0, dest_name.c_str()/*, do_full_copy ? ULONGLONG_MAX:UNIV_PAGE_SIZE */);
+	}
+}
+
 /* ================= prepare ================= */
 
 /***********************************************************************
@@ -4739,6 +5042,27 @@ error:
 	return FALSE;
 }
 
+
+std::string change_extension(std::string filename, std::string new_ext) {
+	DBUG_ASSERT(new_ext.size() == 3);
+	std::string new_name(filename);
+	new_name.resize(new_name.size() - new_ext.size());
+	new_name.append(new_ext);
+	return new_name;
+}
+
+
+static void rename_file(const char *from,const char *to) {
+	msg("Renaming %s to %s\n", from, to);
+	if (my_rename(from, to, MY_WME)) {
+		msg("Cannot rename %s to %s errno %d", from, to, errno);
+		exit(EXIT_FAILURE);
+	}
+}
+
+static void rename_file(const std::string& from, const std::string &to) {
+	rename_file(from.c_str(), to.c_str());
+}
 /************************************************************************
 Callback to handle datadir entry. Function of this type will be called
 for each entry which matches the mask by xb_process_datadir.
@@ -4750,6 +5074,37 @@ typedef ibool (*handle_datadir_entry_func_t)(
 	const char*	file_name,		/*!<in: file name with suffix */
 	void*		arg);			/*!<in: caller-provided data */
 
+/** Rename, and replace destination file, if exists */
+static void rename_force(const char *from, const char *to) {
+	if (access(to, R_OK) == 0) {
+		msg("Removing %s\n", to);
+		if (my_delete(to, MYF(MY_WME))) {
+			msg("Can't remove %s, errno %d", to, errno);
+			exit(EXIT_FAILURE);
+		}
+	}
+	rename_file(from,to);
+}
+
+/* During prepare phase, rename ".new" files , that were created in backup_fix_ddl(),
+  to ".ibd".*/
+static ibool prepare_handle_new_files(
+	const char*	data_home_dir,		/*!<in: path to datadir */
+	const char*	db_name,		/*!<in: database name */
+	const char*	file_name,		/*!<in: file name with suffix */
+	void *)
+{
+
+	std::string src_path = std::string(data_home_dir) + '/' + std::string(db_name) + '/' + file_name;
+	std::string dest_path = src_path;
+
+	size_t index = dest_path.find(".new");
+	DBUG_ASSERT(index != std::string::npos);
+	dest_path.replace(index, 4, ".ibd");
+	rename_force(src_path.c_str(),dest_path.c_str());
+	return TRUE;
+}
+
 /************************************************************************
 Callback to handle datadir entry. Deletes entry if it has no matching
 fil_space in fil_system directory.
@@ -4955,6 +5310,103 @@ store_binlog_info(const char* filename, const char* name, ulonglong pos)
 	return(true);
 }
 
+/** Check if file exists*/
+static bool file_exists(std::string name)
+{
+	return access(name.c_str(), R_OK) == 0 ;
+}
+
+/** Read file content into STL string */
+static std::string read_file_as_string(const std::string file) {
+	char content[FN_REFLEN];
+	FILE *f = fopen(file.c_str(), "r");
+	if (!f) {
+		msg("Can not open %s\n", file.c_str());
+	}
+	size_t len = fread(content, 1, FN_REFLEN, f);
+	fclose(f);
+	return std::string(content, len);
+}
+
+/** Delete file- Provide verbose diagnostics and exit, if operation fails. */
+static void delete_file(const std::string& file, bool if_exists = false) {
+	if (if_exists && !file_exists(file))
+		return;
+	if (my_delete(file.c_str(), MYF(MY_WME))) {
+		msg("Can't remove %s, errno %d", file.c_str(), errno);
+		exit(EXIT_FAILURE);
+	}
+}
+
+/**
+Rename tablespace during prepare.
+Backup in its end phase may generate some .ren files, recording
+tablespaces that should be renamed in --prepare.
+*/
+static void rename_table_in_prepare(const std::string &datadir, const std::string& from , const std::string& to,
+	const char *extension=0) {
+	if (!extension) {
+		static const char *extensions_nonincremental[] = { ".ibd", 0 };
+		static const char *extensions_incremental[] = { ".ibd.delta", ".ibd.meta", 0 };
+		const char **extensions = xtrabackup_incremental_dir ?
+			extensions_incremental : extensions_nonincremental;
+		for (size_t i = 0; extensions[i]; i++) {
+			rename_table_in_prepare(datadir, from, to, extensions[i]);
+		}
+		return;
+	}
+	std::string src = std::string(datadir) + "/" + from + extension;
+	std::string dest = std::string(datadir) + "/" + to + extension;
+	std::string ren2, tmp;
+	if (file_exists(dest)) {
+		ren2= std::string(datadir) + "/" + to + ".ren";
+		if (!file_exists(ren2)) {
+			msg("ERROR : File %s was not found, but expected during rename processing\n", ren2.c_str());
+			ut_a(0);
+		}
+		tmp = to + "#";
+		rename_table_in_prepare(datadir, to, tmp);
+	}
+	rename_file(src, dest);
+	if (ren2.size()) {
+		// Make sure the temp. renamed file is processed.
+		std::string to2 = read_file_as_string(ren2);
+		rename_table_in_prepare(datadir, tmp, to2);
+		delete_file(ren2);
+	}
+}
+
+static ibool prepare_handle_ren_files(const char *datadir, const char *db, const char *filename, void *) {
+
+	std::string ren_file = std::string(datadir) + "/" + db + "/" + filename;
+	if (!file_exists(ren_file))
+		return TRUE;
+
+	std::string to = read_file_as_string(ren_file);
+	std::string source_space_name = std::string(db) + "/" + filename;
+	source_space_name.resize(source_space_name.size() - 4); // remove extension
+
+	rename_table_in_prepare(datadir, source_space_name.c_str(), to.c_str());
+	delete_file(ren_file);
+	return TRUE;
+}
+
+/* Remove tablespaces during backup, based on */
+static ibool prepare_handle_del_files(const char *datadir, const char *db, const char *filename, void *) {
+	std::string del_file = std::string(datadir) + "/" + db + "/" + filename;
+	std::string path(del_file);
+	path.resize(path.size() - 4); // remove extension;
+	if (xtrabackup_incremental) {
+		delete_file(path + ".ibd.delta", true);
+		delete_file(path + ".ibd.meta", true);
+	}
+	else {
+		delete_file(path + ".ibd", true);
+	}
+	delete_file(del_file);
+	return TRUE;
+}
+
 /** Implement --prepare
 @return	whether the operation succeeded */
 static bool
@@ -4972,6 +5424,21 @@ xtrabackup_prepare_func(char** argv)
 	}
 	msg("mariabackup: cd to %s\n", xtrabackup_real_target_dir);
 
+	fil_path_to_mysql_datadir = ".";
+
+	if (xtrabackup_incremental_dir) {
+		xb_process_datadir(xtrabackup_incremental_dir, ".new.meta", prepare_handle_new_files);
+		xb_process_datadir(xtrabackup_incremental_dir, ".new.delta", prepare_handle_new_files);
+	}
+	else {
+		xb_process_datadir(".", ".new", prepare_handle_new_files);
+	}
+	xb_process_datadir(xtrabackup_incremental_dir? xtrabackup_incremental_dir:".",
+		".ren", prepare_handle_ren_files);
+	xb_process_datadir(xtrabackup_incremental_dir ? xtrabackup_incremental_dir : ".",
+		".del", prepare_handle_del_files);
+
+
 	int argc; for (argc = 0; argv[argc]; argc++) {}
 	encryption_plugin_prepare_init(argc, argv);
 
@@ -5334,7 +5801,7 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server)
 	srv_operation = SRV_OPERATION_RESTORE;
 
 	files_charset_info = &my_charset_utf8_general_ci;
-	check_if_backup_includes = backup_includes;
+
 
 	setup_error_messages();
 	sys_var_init();
diff --git a/mysql-test/suite/mariabackup/create_during_backup.result b/mysql-test/suite/mariabackup/create_during_backup.result
new file mode 100644
index 00000000000..f99c2633ab5
--- /dev/null
+++ b/mysql-test/suite/mariabackup/create_during_backup.result
@@ -0,0 +1,10 @@
+# xtrabackup backup
+# xtrabackup prepare
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+SELECT COUNT(*) from t1;
+COUNT(*)
+10000
+DROP TABLE t1;
diff --git a/mysql-test/suite/mariabackup/create_during_backup.test b/mysql-test/suite/mariabackup/create_during_backup.test
new file mode 100644
index 00000000000..985a5a3e53a
--- /dev/null
+++ b/mysql-test/suite/mariabackup/create_during_backup.test
@@ -0,0 +1,26 @@
+--source include/have_debug.inc
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+mkdir $targetdir;
+
+# this will table and populate it, after backup has list of tables to be copied
+--let after_load_tablespaces =CREATE TABLE test.t1 ENGINE=INNODB SELECT UUID() from test.seq_1_to_10000
+
+echo # xtrabackup backup;
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup  --target-dir=$targetdir  --dbug=+d,mariabackup_events;
+--enable_result_log
+
+--let after_load_tables=
+
+echo # xtrabackup prepare;
+--disable_result_log
+exec $XTRABACKUP  --prepare --target-dir=$targetdir;
+-- source include/restart_and_restore.inc
+--enable_result_log
+
+# Check that new table is there after restore.
+SELECT COUNT(*) from t1;
+DROP TABLE t1;
+rmdir $targetdir;
+
+
diff --git a/mysql-test/suite/mariabackup/create_with_data_directory_during_backup.result b/mysql-test/suite/mariabackup/create_with_data_directory_during_backup.result
new file mode 100644
index 00000000000..a06150bcb6a
--- /dev/null
+++ b/mysql-test/suite/mariabackup/create_with_data_directory_during_backup.result
@@ -0,0 +1,10 @@
+# xtrabackup backup
+# xtrabackup prepare
+DROP TABLE t;
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+SELECT * FROM t;
+i
+DROP TABLE t;
diff --git a/mysql-test/suite/mariabackup/create_with_data_directory_during_backup.test b/mysql-test/suite/mariabackup/create_with_data_directory_during_backup.test
new file mode 100644
index 00000000000..d0cb83d069f
--- /dev/null
+++ b/mysql-test/suite/mariabackup/create_with_data_directory_during_backup.test
@@ -0,0 +1,24 @@
+--source include/have_debug.inc
+let $table_data_dir=$MYSQLTEST_VARDIR/tmp/ddir;
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+mkdir $table_data_dir;
+--replace_result $table_data_dir table_data_dir
+--let after_load_tablespaces=CREATE TABLE test.t(i int) ENGINE=INNODB DATA DIRECTORY='$table_data_dir'
+echo # xtrabackup backup;
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+--disable_result_log
+exec   $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --dbug=+d,mariabackup_events;
+--enable_result_log
+--source include/shutdown_mysqld.inc
+echo # xtrabackup prepare;
+--disable_result_log
+exec $XTRABACKUP  --prepare --target-dir=$targetdir;
+--source include/start_mysqld.inc
+DROP TABLE t;
+rmdir $table_data_dir;
+-- source include/restart_and_restore.inc
+--enable_result_log
+SELECT * FROM t;
+DROP TABLE t;
+rmdir $targetdir;
+rmdir $table_data_dir;
diff --git a/mysql-test/suite/mariabackup/disabled.def b/mysql-test/suite/mariabackup/disabled.def
index 8b137891791..a7be04bfc16 100644
--- a/mysql-test/suite/mariabackup/disabled.def
+++ b/mysql-test/suite/mariabackup/disabled.def
@@ -1 +1 @@
-
+unsupported_redo    : MDEV-16791 allows optimized redo
\ No newline at end of file
diff --git a/mysql-test/suite/mariabackup/drop_table_during_backup.result b/mysql-test/suite/mariabackup/drop_table_during_backup.result
new file mode 100644
index 00000000000..1472ddbddcf
--- /dev/null
+++ b/mysql-test/suite/mariabackup/drop_table_during_backup.result
@@ -0,0 +1,13 @@
+CREATE TABLE t1 (i int) ENGINE=INNODB;
+CREATE TABLE t2 (i int) ENGINE=INNODB;
+CREATE TABLE t3 (i int) ENGINE=INNODB;
+# xtrabackup prepare
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+CREATE TABLE t1(i int);
+DROP TABLE t1;
+CREATE TABLE t2(i int);
+DROP TABLE t2;
+DROP TABLE t3;
diff --git a/mysql-test/suite/mariabackup/drop_table_during_backup.test b/mysql-test/suite/mariabackup/drop_table_during_backup.test
new file mode 100644
index 00000000000..02c7710145b
--- /dev/null
+++ b/mysql-test/suite/mariabackup/drop_table_during_backup.test
@@ -0,0 +1,24 @@
+--source include/have_debug.inc
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+CREATE TABLE t1 (i int) ENGINE=INNODB;
+CREATE TABLE t2 (i int) ENGINE=INNODB;
+CREATE TABLE t3 (i int) ENGINE=INNODB;
+--let before_copy_test_t1=DROP TABLE test.t1
+--let after_copy_test_t2=DROP TABLE test.t2;
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --dbug=+d,mariabackup_events;
+--enable_result_log
+
+echo # xtrabackup prepare;
+--disable_result_log
+exec $XTRABACKUP  --prepare --target-dir=$targetdir;
+-- source include/restart_and_restore.inc
+--enable_result_log
+
+#check that the table t1 does not exist in backup
+CREATE TABLE t1(i int);
+DROP TABLE t1;
+CREATE TABLE t2(i int);
+DROP TABLE t2;
+DROP TABLE t3;
+rmdir $targetdir;
diff --git a/mysql-test/suite/mariabackup/incremental_ddl_during_backup.result b/mysql-test/suite/mariabackup/incremental_ddl_during_backup.result
new file mode 100644
index 00000000000..ffca1ef0e1f
--- /dev/null
+++ b/mysql-test/suite/mariabackup/incremental_ddl_during_backup.result
@@ -0,0 +1,22 @@
+call mtr.add_suppression("InnoDB: New log files created");
+CREATE TABLE t1(i INT PRIMARY KEY) ENGINE INNODB;
+CREATE TABLE t2(i INT PRIMARY KEY) ENGINE INNODB;
+CREATE TABLE t3(i INT) ENGINE INNODB;
+# Create full backup , modify table, then create incremental/differential backup
+INSERT into t1 values(1);
+# Prepare full backup, apply incremental one
+# Restore and check results
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+CREATE TABLE t1(i int);
+DROP TABLE t1;
+SELECT * from t1_renamed;
+i
+1
+DROP TABLE t1_renamed;
+CREATE TABLE t2(i INT PRIMARY KEY) ENGINE INNODB;
+DROP TABLE t2;
+DROP TABLE t3;
+DROP TABLE t4;
diff --git a/mysql-test/suite/mariabackup/incremental_ddl_during_backup.test b/mysql-test/suite/mariabackup/incremental_ddl_during_backup.test
new file mode 100644
index 00000000000..60780b13920
--- /dev/null
+++ b/mysql-test/suite/mariabackup/incremental_ddl_during_backup.test
@@ -0,0 +1,52 @@
+--source include/have_debug.inc
+
+call mtr.add_suppression("InnoDB: New log files created");
+
+let $basedir=$MYSQLTEST_VARDIR/tmp/backup;
+let $incremental_dir=$MYSQLTEST_VARDIR/tmp/backup_inc1;
+
+CREATE TABLE t1(i INT PRIMARY KEY) ENGINE INNODB;
+CREATE TABLE t2(i INT PRIMARY KEY) ENGINE INNODB;
+CREATE TABLE t3(i INT) ENGINE INNODB;
+
+echo # Create full backup , modify table, then create incremental/differential backup;
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup --target-dir=$basedir;
+--enable_result_log
+
+INSERT into t1 values(1);
+
+--let after_load_tablespaces=CREATE TABLE test.t4 ENGINE=INNODB SELECT UUID() from test.seq_1_to_10000
+--let after_copy_test_t1=RENAME TABLE  test.t1 TO test.t1_renamed
+--let after_copy_test_t2=DROP TABLE test.t2
+--let after_copy_test_t3=CREATE INDEX a_i ON test.t3(i);
+
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup --target-dir=$incremental_dir --incremental-basedir=$basedir --dbug=+d,mariabackup_events;
+--let after_load_tablespaces=
+--disable_result_log
+echo # Prepare full backup, apply incremental one;
+exec $XTRABACKUP --apply-log-only --prepare --target-dir=$basedir;
+exec $XTRABACKUP --prepare --target-dir=$basedir --incremental-dir=$incremental_dir ;
+
+echo # Restore and check results;
+let $targetdir=$basedir;
+-- source include/restart_and_restore.inc
+--enable_result_log
+
+# Test that t1 does not exist, but t1_renamed does
+CREATE TABLE t1(i int);
+DROP TABLE t1;
+
+SELECT * from t1_renamed;
+DROP TABLE t1_renamed;
+
+# Test that t2 does not exist;
+CREATE TABLE t2(i INT PRIMARY KEY) ENGINE INNODB;
+DROP TABLE t2;
+
+DROP TABLE t3;
+DROP TABLE t4;
+
+# Cleanup
+rmdir $basedir;
+rmdir $incremental_dir;
diff --git a/mysql-test/suite/mariabackup/mlog_index_load.result b/mysql-test/suite/mariabackup/mlog_index_load.result
new file mode 100644
index 00000000000..bc83981106e
--- /dev/null
+++ b/mysql-test/suite/mariabackup/mlog_index_load.result
@@ -0,0 +1,15 @@
+CREATE TABLE t1(i INT PRIMARY KEY auto_increment, a int) ENGINE INNODB;
+INSERT INTO t1(a) SELECT * from seq_1_to_10000;
+# xtrabackup backup
+t1.frm
+t1.ibd
+t1.new
+# xtrabackup prepare
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+SELECT COUNT(*) from t1;
+COUNT(*)
+10000
+DROP TABLE t1;
diff --git a/mysql-test/suite/mariabackup/mlog_index_load.test b/mysql-test/suite/mariabackup/mlog_index_load.test
new file mode 100644
index 00000000000..fb29041fd3c
--- /dev/null
+++ b/mysql-test/suite/mariabackup/mlog_index_load.test
@@ -0,0 +1,27 @@
+--source include/have_debug.inc
+
+CREATE TABLE t1(i INT PRIMARY KEY auto_increment, a int) ENGINE INNODB;
+INSERT INTO t1(a) SELECT * from seq_1_to_10000;
+
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+
+let after_copy_test_t1=CREATE INDEX a_ind ON test.t1(a) ALGORITHM=INPLACE;
+echo # xtrabackup backup;
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup  --target-dir=$targetdir  --dbug=+d,mariabackup_events;
+--enable_result_log
+
+--list_files $targetdir/test t1*
+--let before_copy_test_t1=
+
+echo # xtrabackup prepare;
+--disable_result_log
+exec $XTRABACKUP  --prepare --target-dir=$targetdir;
+-- source include/restart_and_restore.inc
+--enable_result_log
+
+# Check that new table is there after restore.
+SELECT COUNT(*) from t1;
+DROP TABLE t1;
+rmdir $targetdir;
+
diff --git a/mysql-test/suite/mariabackup/recreate_table_during_backup.result b/mysql-test/suite/mariabackup/recreate_table_during_backup.result
new file mode 100644
index 00000000000..4e038fee71a
--- /dev/null
+++ b/mysql-test/suite/mariabackup/recreate_table_during_backup.result
@@ -0,0 +1,23 @@
+CREATE TABLE t1(i int) ENGINE=INNODB;
+CREATE TABLE t2(i int) ENGINE=INNODB;
+CREATE TABLE t3(a CHAR(36)) ENGINE INNODB;
+INSERT INTO t3 SELECT UUID() FROM seq_1_to_1000;
+# xtrabackup backup
+# xtrabackup prepare
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+SELECT COUNT(*) from t1;
+COUNT(*)
+100
+SELECT COUNT(*) from t2;
+COUNT(*)
+1000
+SELECT COUNT(*) from t3;
+COUNT(*)
+1000
+DROP INDEX index_a ON t3;
+DROP TABLE t1;
+DROP TABLE t2;
+DROP TABLE t3;
diff --git a/mysql-test/suite/mariabackup/recreate_table_during_backup.test b/mysql-test/suite/mariabackup/recreate_table_during_backup.test
new file mode 100644
index 00000000000..c3c9cf5aeef
--- /dev/null
+++ b/mysql-test/suite/mariabackup/recreate_table_during_backup.test
@@ -0,0 +1,37 @@
+--source include/have_debug.inc
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+mkdir $targetdir;
+
+CREATE TABLE t1(i int) ENGINE=INNODB;
+CREATE TABLE t2(i int) ENGINE=INNODB;
+CREATE TABLE t3(a CHAR(36)) ENGINE INNODB;
+INSERT INTO t3 SELECT UUID() FROM seq_1_to_1000;
+
+# this will table and populate it, after backup has list of tables to be copied
+--let before_copy_test_t1=BEGIN NOT ATOMIC DROP TABLE test.t1;CREATE TABLE test.t1 ENGINE=INNODB SELECT UUID() from test.seq_1_to_100; END
+--let after_copy_test_t2=BEGIN NOT ATOMIC  DROP TABLE test.t2;CREATE TABLE test.t2 ENGINE=INNODB SELECT UUID() from test.seq_1_to_1000; END
+--let after_copy_test_t3=ALTER TABLE test.t3 ADD INDEX index_a(a),ALGORITHM=COPY
+echo # xtrabackup backup;
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup  --target-dir=$targetdir  --close-files --dbug=+d,mariabackup_events;
+--enable_result_log
+--let after_load_tables=
+
+echo # xtrabackup prepare;
+--disable_result_log
+exec $XTRABACKUP  --prepare --target-dir=$targetdir;
+-- source include/restart_and_restore.inc
+--enable_result_log
+
+# Check that new table is there after restore.
+SELECT COUNT(*) from t1;
+SELECT COUNT(*) from t2;
+SELECT COUNT(*) from t3;
+DROP INDEX index_a ON t3;
+DROP TABLE t1;
+DROP TABLE t2;
+DROP TABLE t3;
+rmdir $targetdir;
+
+
+
diff --git a/mysql-test/suite/mariabackup/rename_during_backup.result b/mysql-test/suite/mariabackup/rename_during_backup.result
new file mode 100644
index 00000000000..6e8f365f643
--- /dev/null
+++ b/mysql-test/suite/mariabackup/rename_during_backup.result
@@ -0,0 +1,53 @@
+CREATE TABLE t1(i int)   ENGINE INNODB;
+INSERT into t1 values(1);
+CREATE TABLE t2(i int)   ENGINE INNODB;
+INSERT INTO t2 values(2);
+CREATE TABLE t3(i int) ENGINE INNODB;
+CREATE TABLE t4(i int) ENGINE INNODB;
+CREATE TABLE a(a int)    ENGINE INNODB;
+INSERT INTO a values(1);
+CREATE TABLE b(b CHAR(1)) ENGINE INNODB;
+INSERT INTO b VALUES('b');
+CREATE TABLE a1(a1 int)    ENGINE INNODB;
+INSERT INTO a1 VALUES(1);
+CREATE TABLE b1(b1 CHAR(2)) ENGINE INNODB;
+INSERT INTO b1 VALUES('b1');
+# xtrabackup prepare
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+CREATE TABLE t1(i int);
+DROP TABLE t1;
+SELECT * from t1_renamed;
+i
+1
+DROP TABLE t1_renamed;
+CREATE TABLE t2(i int);
+DROP TABLE t2;
+SELECT * from t2_renamed;
+i
+2
+DROP TABLE t2_renamed;
+SELECT * from t3;
+i
+3
+DROP TABLE t3;
+SELECT * from t4;
+i
+DROP TABLE t4;
+CREATE TABLE tmp(i int);
+DROP TABLE tmp;
+SELECT * FROM a;
+b
+b
+SELECT * FROM b;
+a
+1
+SELECT * FROM a1;
+b1
+b1
+SELECT * FROM b1;
+a1
+1
+DROP TABLE a,b,a1,b1;
diff --git a/mysql-test/suite/mariabackup/rename_during_backup.test b/mysql-test/suite/mariabackup/rename_during_backup.test
new file mode 100644
index 00000000000..f176af1451b
--- /dev/null
+++ b/mysql-test/suite/mariabackup/rename_during_backup.test
@@ -0,0 +1,86 @@
+--source include/have_debug.inc
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+mkdir $targetdir;
+
+CREATE TABLE t1(i int)   ENGINE INNODB;
+INSERT into t1 values(1);
+
+CREATE TABLE t2(i int)   ENGINE INNODB;
+INSERT INTO t2 values(2);
+
+CREATE TABLE t3(i int) ENGINE INNODB;
+CREATE TABLE t4(i int) ENGINE INNODB;
+
+CREATE TABLE a(a int)    ENGINE INNODB;
+INSERT INTO a values(1);
+CREATE TABLE b(b CHAR(1)) ENGINE INNODB;
+INSERT INTO b VALUES('b');
+
+CREATE TABLE a1(a1 int)    ENGINE INNODB;
+INSERT INTO a1 VALUES(1);
+
+CREATE TABLE b1(b1 CHAR(2)) ENGINE INNODB;
+INSERT INTO b1 VALUES('b1');
+
+# Test renames before of after copying tablespaces
+--let before_copy_test_t1=RENAME TABLE test.t1 TO test.t1_renamed
+--let after_copy_test_t2=RENAME TABLE  test.t2 TO test.t2_renamed
+--let after_copy_test_t3=BEGIN NOT ATOMIC RENAME TABLE test.t3 TO test.t3_tmp; INSERT INTO test.t3_tmp VALUES(3); RENAME TABLE test.t3_tmp TO test.t3; END
+--let before_copy_test_t4=RENAME TABLE test.t4 TO test.t4_tmp
+--let after_copy_test_t4=RENAME TABLE test.t4_tmp TO test.t4
+
+# Test circular renames
+--let before_copy_test_b=RENAME TABLE test.a to test.tmp, test.b to test.a, test.tmp to test.b
+--let after_copy_test_b1=RENAME TABLE test.a1 to test.tmp, test.b1 to test.a1, test.tmp to test.b1
+
+--disable_result_log 
+exec  $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup --target-dir=$targetdir  --dbug=+d,mariabackup_events;
+--enable_result_log
+
+--let before_copy_test_t1=''
+--let after_copy_test_t2=''
+--let before_copy_test_a=''
+--let after_copy_test_a1=''
+
+echo # xtrabackup prepare;
+--disable_result_log
+exec $XTRABACKUP  --prepare --target-dir=$targetdir;
+-- source include/restart_and_restore.inc
+--enable_result_log
+
+# the table was renamed from t1 to t1_renamed
+# make sure t1 does not exist, and t1_renamed does.
+
+CREATE TABLE t1(i int);
+DROP TABLE t1;
+
+SELECT * from t1_renamed;
+DROP TABLE t1_renamed;
+
+
+CREATE TABLE t2(i int);
+DROP TABLE t2;
+
+SELECT * from t2_renamed;
+DROP TABLE t2_renamed;
+
+#rename to itself
+SELECT * from t3;
+DROP TABLE t3;
+
+SELECT * from t4;
+DROP TABLE t4;
+
+# For circular renames , make sure intermediate tables do not exist
+CREATE TABLE tmp(i int);
+DROP TABLE tmp;
+
+SELECT * FROM a;
+SELECT * FROM b;
+SELECT * FROM a1;
+SELECT * FROM b1;
+
+DROP TABLE a,b,a1,b1;
+rmdir $targetdir;
+
+
diff --git a/mysql-test/suite/mariabackup/rename_during_mdl_lock.result b/mysql-test/suite/mariabackup/rename_during_mdl_lock.result
index 982851438f2..3b64cdc38bc 100644
--- a/mysql-test/suite/mariabackup/rename_during_mdl_lock.result
+++ b/mysql-test/suite/mariabackup/rename_during_mdl_lock.result
@@ -1,3 +1,11 @@
 CREATE TABLE t1(i int) ENGINE INNODB;
-FOUND 1 /failed to execute query SELECT 1 FROM/ in backup.log
+# xtrabackup prepare
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+CREATE TABLE t1(i int);
+DROP TABLE t1;
+SELECT * from t2;
+i
 DROP TABLE t2;
diff --git a/mysql-test/suite/mariabackup/rename_during_mdl_lock.test b/mysql-test/suite/mariabackup/rename_during_mdl_lock.test
index 0a41f1dfe74..6d22e0db4a7 100644
--- a/mysql-test/suite/mariabackup/rename_during_mdl_lock.test
+++ b/mysql-test/suite/mariabackup/rename_during_mdl_lock.test
@@ -1,13 +1,21 @@
 --source include/have_debug.inc
-let $targetdir=$MYSQLTEST_VARDIR/backup;
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
 mkdir $targetdir;
 CREATE TABLE t1(i int) ENGINE INNODB;
---error 1
-exec  $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup  --target-dir=$targetdir  --lock-ddl-per-table --dbug=+d,rename_during_mdl_lock_table 2>$targetdir/backup.log;
+exec  $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup  --target-dir=$targetdir  --lock-ddl-per-table --dbug=+d,rename_during_mdl_lock_table;
 
-let SEARCH_FILE=$targetdir/backup.log;
-let SEARCH_PATTERN=failed to execute query SELECT 1 FROM;
-source include/search_pattern_in_file.inc;
+echo # xtrabackup prepare;
+--disable_result_log
+exec $XTRABACKUP  --prepare --target-dir=$targetdir;
+-- source include/restart_and_restore.inc
+--enable_result_log
+# the table was renamed from t1 to t2
+# make sure t1 does not exist, and t2 does
 
+CREATE TABLE t1(i int);
+DROP TABLE t1;
+SELECT * from t2;
 DROP TABLE t2;
+
 rmdir $targetdir;
+
diff --git a/mysql-test/suite/mariabackup/suite.opt b/mysql-test/suite/mariabackup/suite.opt
index de3637814b2..1df4643562e 100644
--- a/mysql-test/suite/mariabackup/suite.opt
+++ b/mysql-test/suite/mariabackup/suite.opt
@@ -1 +1 @@
---innodb --loose-changed_page_bitmaps --innodb-sys-tables --innodb-flush-log-at-trx-commit=2
+--innodb --loose-changed_page_bitmaps --innodb-sys-tables --innodb-flush-log-at-trx-commit=2 --sequence
diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h
index 6dcaaedd765..6716b1cbd0d 100644
--- a/storage/innobase/include/log0recv.h
+++ b/storage/innobase/include/log0recv.h
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation.
+Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -153,9 +153,21 @@ bool recv_parse_log_recs(lsn_t checkpoint_lsn, store_t store, bool apply);
 /** Moves the parsing buffer data left to the buffer start. */
 void recv_sys_justify_left_parsing_buf();
 
-/** Backup function checks whether the space id belongs to
-the skip table list given in the mariabackup option. */
-extern bool(*check_if_backup_includes)(ulint space_id);
+/** Report optimized DDL operation (without redo log), corresponding to MLOG_INDEX_LOAD.
+@param[in]	space_id	tablespace identifier
+*/
+extern void(*log_optimized_ddl_op)(ulint space_id);
+
+/** Report an operation to create, delete, or rename a file during backup.
+@param[in]	space_id	tablespace identifier
+@param[in]	flags		tablespace flags (NULL if not create)
+@param[in]	name		file name (not NUL-terminated)
+@param[in]	len		length of name, in bytes
+@param[in]	new_name	new file name (NULL if not rename)
+@param[in]	new_len		length of new_name, in bytes (0 if NULL) */
+extern void (*log_file_op)(ulint space_id, const byte* flags,
+			   const byte* name, ulint len,
+			   const byte* new_name, ulint new_len);
 
 /** Block of log record data */
 struct recv_data_t{
diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc
index 45f0be93d4e..77e61606c1e 100644
--- a/storage/innobase/log/log0recv.cc
+++ b/storage/innobase/log/log0recv.cc
@@ -169,9 +169,21 @@ typedef std::map<
 
 static recv_spaces_t	recv_spaces;
 
-/** Backup function checks whether the space id belongs to
-the skip table list given in the mariabackup option. */
-bool(*check_if_backup_includes)(ulint space_id);
+/** Report optimized DDL operation (without redo log), corresponding to MLOG_INDEX_LOAD.
+@param[in]	space_id	tablespace identifier
+*/
+void (*log_optimized_ddl_op)(ulint space_id);
+
+/** Report an operation to create, delete, or rename a file during backup.
+@param[in]	space_id	tablespace identifier
+@param[in]	flags		tablespace flags (NULL if not create)
+@param[in]	name		file name (not NUL-terminated)
+@param[in]	len		length of name, in bytes
+@param[in]	new_name	new file name (NULL if not rename)
+@param[in]	new_len		length of new_name, in bytes (0 if NULL) */
+void (*log_file_op)(ulint space_id, const byte* flags,
+		    const byte* name, ulint len,
+		    const byte* new_name, ulint new_len);
 
 /** Process a file name from a MLOG_FILE_* record.
 @param[in,out]	name		file name
@@ -381,9 +393,13 @@ fil_name_parse(
 
 		fil_name_process(
 			reinterpret_cast<char*>(ptr), len, space_id, true);
-
-		break;
+		/* fall through */
 	case MLOG_FILE_CREATE2:
+		if (log_file_op) {
+			log_file_op(space_id,
+				    type == MLOG_FILE_CREATE2 ? ptr - 4 : NULL,
+				    ptr, len, NULL, 0);
+		}
 		break;
 	case MLOG_FILE_RENAME2:
 		if (corrupt) {
@@ -424,6 +440,11 @@ fil_name_parse(
 			reinterpret_cast<char*>(new_name), new_len,
 			space_id, false);
 
+		if (log_file_op) {
+			log_file_op(space_id, NULL,
+				    ptr, len, new_name, new_len);
+		}
+
 		if (!apply) {
 			break;
 		}
@@ -2503,11 +2524,8 @@ loop:
 			/* fall through */
 		case MLOG_INDEX_LOAD:
 			if (type == MLOG_INDEX_LOAD) {
-				if (check_if_backup_includes
-				    && !check_if_backup_includes(space)) {
-					ut_ad(srv_operation
-					      == SRV_OPERATION_BACKUP);
-					return true;
+				if (log_optimized_ddl_op) {
+					log_optimized_ddl_op(space);
 				}
 			}
 			/* fall through */

From 964ad0c426926034933eb759d34de7379dbf4aed Mon Sep 17 00:00:00 2001
From: Daniel Bartholomew <db@dbart.us>
Date: Tue, 14 Aug 2018 12:32:44 -0400
Subject: [PATCH 155/164] bump the VERSION

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index e066e232278..e9c66ed8d77 100644
--- a/VERSION
+++ b/VERSION
@@ -1,3 +1,3 @@
 MYSQL_VERSION_MAJOR=10
 MYSQL_VERSION_MINOR=2
-MYSQL_VERSION_PATCH=17
+MYSQL_VERSION_PATCH=18

From 021652ba5035c0bcf1da02fb81b59b88893f09bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Tue, 14 Aug 2018 19:34:33 +0300
Subject: [PATCH 156/164] MDEV-15872 Crash in online ALTER TABLE...ADD PRIMARY
 KEY after instant ADD COLUMN...NULL

row_log_table_get_pk_col(): Replace a condition that was inadvertently
removed in MDEV-16365. PRIMARY KEY columns are never allowed to be NULL,
and failure to enforce the constraint caused a null pointer to be
dereferenced in mem_heap_dup().
---
 .../suite/innodb/r/instant_alter_debug.result | 25 +++++++++++++++++-
 .../suite/innodb/t/instant_alter_debug.test   | 26 +++++++++++++++++++
 storage/innobase/row/row0log.cc               |  4 ++-
 3 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/mysql-test/suite/innodb/r/instant_alter_debug.result b/mysql-test/suite/innodb/r/instant_alter_debug.result
index 3c0a1600029..72e42763569 100644
--- a/mysql-test/suite/innodb/r/instant_alter_debug.result
+++ b/mysql-test/suite/innodb/r/instant_alter_debug.result
@@ -205,12 +205,35 @@ INSERT INTO t1 SET a=1;
 INSERT INTO t1 SET a=2,b=3,c=4;
 SET DEBUG_SYNC = 'now SIGNAL logged';
 connection ddl;
-disconnect ddl;
 connection default;
 SET DEBUG_SYNC = RESET;
 SELECT * FROM t1;
 a	b	c
 1	2	NULL
 2	3	4
+#
+# MDEV-15872 Crash in online ALTER TABLE...ADD PRIMARY KEY
+# after instant ADD COLUMN ... NULL
+#
+ALTER TABLE t1 ADD COLUMN d INT, ALGORITHM=INSTANT;
+UPDATE t1 SET d=1;
+connection ddl;
+SET DEBUG_SYNC='row_log_table_apply1_before SIGNAL copied WAIT_FOR logged';
+ALTER IGNORE TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY (a,d);
+connection default;
+SET DEBUG_SYNC = 'now WAIT_FOR copied';
+BEGIN;
+INSERT INTO t1 SET a=3;
+ROLLBACK;
+SET DEBUG_SYNC = 'now SIGNAL logged';
+connection ddl;
+ERROR 22004: Invalid use of NULL value
+disconnect ddl;
+connection default;
+SET DEBUG_SYNC = RESET;
+SELECT * FROM t1;
+a	b	c	d
+1	2	NULL	1
+2	3	4	1
 DROP TABLE t1;
 SET GLOBAL innodb_purge_rseg_truncate_frequency = @save_frequency;
diff --git a/mysql-test/suite/innodb/t/instant_alter_debug.test b/mysql-test/suite/innodb/t/instant_alter_debug.test
index 5a73ed65d72..cab9decd3d0 100644
--- a/mysql-test/suite/innodb/t/instant_alter_debug.test
+++ b/mysql-test/suite/innodb/t/instant_alter_debug.test
@@ -234,6 +234,32 @@ SET DEBUG_SYNC = 'now SIGNAL logged';
 
 connection ddl;
 reap;
+
+connection default;
+SET DEBUG_SYNC = RESET;
+SELECT * FROM t1;
+
+--echo #
+--echo # MDEV-15872 Crash in online ALTER TABLE...ADD PRIMARY KEY
+--echo # after instant ADD COLUMN ... NULL
+--echo #
+ALTER TABLE t1 ADD COLUMN d INT, ALGORITHM=INSTANT;
+UPDATE t1 SET d=1;
+
+connection ddl;
+SET DEBUG_SYNC='row_log_table_apply1_before SIGNAL copied WAIT_FOR logged';
+send ALTER IGNORE TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY (a,d);
+
+connection default;
+SET DEBUG_SYNC = 'now WAIT_FOR copied';
+BEGIN;
+INSERT INTO t1 SET a=3;
+ROLLBACK;
+SET DEBUG_SYNC = 'now SIGNAL logged';
+
+connection ddl;
+--error ER_INVALID_USE_OF_NULL
+reap;
 disconnect ddl;
 
 connection default;
diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc
index 386bd2786a0..a6e35744446 100644
--- a/storage/innobase/row/row0log.cc
+++ b/storage/innobase/row/row0log.cc
@@ -1164,7 +1164,6 @@ row_log_table_get_pk_col(
 	field = rec_get_nth_field(rec, offsets, i, &len);
 
 	if (len == UNIV_SQL_NULL) {
-
 		if (!log->allow_not_null) {
 			return(DB_INVALID_NULL);
 		}
@@ -1173,6 +1172,9 @@ row_log_table_get_pk_col(
 
 		field = static_cast<const byte*>(
 			log->defaults->fields[n_default_cols].data);
+		if (!field) {
+			return(DB_INVALID_NULL);
+		}
 		len = log->defaults->fields[i - DATA_N_SYS_COLS].len;
 	}
 

From b795134802c486311e1987f18a95fb17c12935e3 Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Tue, 14 Aug 2018 18:58:41 +0200
Subject: [PATCH 157/164] MDEV-16859  MyRocks: support SSE42 CRC32-C
 instruction.

---
 storage/rocksdb/build_rocksdb.cmake | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/storage/rocksdb/build_rocksdb.cmake b/storage/rocksdb/build_rocksdb.cmake
index 0f1b77e255d..895d473c2d4 100644
--- a/storage/rocksdb/build_rocksdb.cmake
+++ b/storage/rocksdb/build_rocksdb.cmake
@@ -376,9 +376,31 @@ SET(SOURCES)
 FOREACH(s ${ROCKSDB_SOURCES})
   list(APPEND SOURCES ${ROCKSDB_SOURCE_DIR}/${s})
 ENDFOREACH()
-IF(MSVC)
+
+if(MSVC)
   add_definitions(-DHAVE_SSE42 -DHAVE_PCLMUL)
-ENDIF()
+else()
+  set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul ${CXX11_FLAGS}")
+
+  CHECK_CXX_SOURCE_COMPILES("
+#include <cstdint>
+#include <nmmintrin.h>
+#include <wmmintrin.h>
+int main() {
+  volatile uint32_t x = _mm_crc32_u32(0, 0);
+  const auto a = _mm_set_epi64x(0, 0);
+  const auto b = _mm_set_epi64x(0, 0);
+  const auto c = _mm_clmulepi64_si128(a, b, 0x00);
+  auto d = _mm_cvtsi128_si64(c);
+}
+" HAVE_SSE42)
+  if(HAVE_SSE42)
+    set_source_files_properties(${ROCKSDB_SOURCE_DIR}/util/crc32c.cc
+      PROPERTIES COMPILE_FLAGS "-DHAVE_SSE42 -DHAVE_PCLMUL -msse4.2 -mpclmul")
+  endif()
+  unset(CMAKE_REQUIRED_FLAGS)
+endif()
+
 IF(CMAKE_VERSION VERSION_GREATER "2.8.10")
   STRING(TIMESTAMP GIT_DATE_TIME "%Y-%m-%d %H:%M:%S")
 ENDIF()

From 197aa0d879c70861d39d1aa6720394ebbaa0274a Mon Sep 17 00:00:00 2001
From: Daniel Bartholomew <db@dbart.us>
Date: Wed, 15 Aug 2018 10:32:52 -0400
Subject: [PATCH 158/164] bump the VERSION

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index e47719133ed..177dfcacb9d 100644
--- a/VERSION
+++ b/VERSION
@@ -1,4 +1,4 @@
 MYSQL_VERSION_MAJOR=10
 MYSQL_VERSION_MINOR=3
-MYSQL_VERSION_PATCH=9
+MYSQL_VERSION_PATCH=10
 SERVER_MATURITY=stable

From 3ce8a0fc494ef8d1561cde4bf3a467d465e0c70e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Mon, 13 Aug 2018 11:46:22 +0300
Subject: [PATCH 159/164] MDEV-16136: Simplify trx_lock_t memory management

Allocate trx->lock.rec_pool and trx->lock.table_pool directly from trx_t.
Remove unnecessary use of std::vector.

In order to do this, move some definitions from lock0priv.h to
lock0types.h, so that ib_lock_t will not be an opaque type.
---
 storage/innobase/include/lock0lock.h  |  63 ---------
 storage/innobase/include/lock0priv.h  | 139 +------------------
 storage/innobase/include/lock0types.h | 192 +++++++++++++++++++++++++-
 storage/innobase/include/trx0trx.h    |  14 +-
 storage/innobase/lock/lock0lock.cc    |  50 +------
 storage/innobase/trx/trx0trx.cc       |  33 +----
 6 files changed, 216 insertions(+), 275 deletions(-)

diff --git a/storage/innobase/include/lock0lock.h b/storage/innobase/include/lock0lock.h
index c9500f7fd49..f7d507813e3 100644
--- a/storage/innobase/include/lock0lock.h
+++ b/storage/innobase/include/lock0lock.h
@@ -834,69 +834,6 @@ lock_trx_has_rec_x_lock(
 	MY_ATTRIBUTE((warn_unused_result));
 #endif /* UNIV_DEBUG */
 
-/**
-Allocate cached locks for the transaction.
-@param trx		allocate cached record locks for this transaction */
-void
-lock_trx_alloc_locks(trx_t* trx);
-
-/** Lock modes and types */
-/* @{ */
-#define LOCK_MODE_MASK	0xFUL	/*!< mask used to extract mode from the
-				type_mode field in a lock */
-/** Lock types */
-/* @{ */
-#define LOCK_TABLE	16U	/*!< table lock */
-#define	LOCK_REC	32U	/*!< record lock */
-#define LOCK_TYPE_MASK	0xF0UL	/*!< mask used to extract lock type from the
-				type_mode field in a lock */
-#if LOCK_MODE_MASK & LOCK_TYPE_MASK
-# error "LOCK_MODE_MASK & LOCK_TYPE_MASK"
-#endif
-
-#define LOCK_WAIT	256U	/*!< Waiting lock flag; when set, it
-				means that the lock has not yet been
-				granted, it is just waiting for its
-				turn in the wait queue */
-/* Precise modes */
-#define LOCK_ORDINARY	0	/*!< this flag denotes an ordinary
-				next-key lock in contrast to LOCK_GAP
-				or LOCK_REC_NOT_GAP */
-#define LOCK_GAP	512U	/*!< when this bit is set, it means that the
-				lock holds only on the gap before the record;
-				for instance, an x-lock on the gap does not
-				give permission to modify the record on which
-				the bit is set; locks of this type are created
-				when records are removed from the index chain
-				of records */
-#define LOCK_REC_NOT_GAP 1024U	/*!< this bit means that the lock is only on
-				the index record and does NOT block inserts
-				to the gap before the index record; this is
-				used in the case when we retrieve a record
-				with a unique key, and is also used in
-				locking plain SELECTs (not part of UPDATE
-				or DELETE) when the user has set the READ
-				COMMITTED isolation level */
-#define LOCK_INSERT_INTENTION 2048U/*!< this bit is set when we place a waiting
-				gap type record lock request in order to let
-				an insert of an index record to wait until
-				there are no conflicting locks by other
-				transactions on the gap; note that this flag
-				remains set when the waiting lock is granted,
-				or if the lock is inherited to a neighboring
-				record */
-#define LOCK_PREDICATE	8192U	/*!< Predicate lock */
-#define LOCK_PRDT_PAGE	16384U	/*!< Page lock */
-
-
-#if (LOCK_WAIT|LOCK_GAP|LOCK_REC_NOT_GAP|LOCK_INSERT_INTENTION|LOCK_PREDICATE|LOCK_PRDT_PAGE)&LOCK_MODE_MASK
-# error
-#endif
-#if (LOCK_WAIT|LOCK_GAP|LOCK_REC_NOT_GAP|LOCK_INSERT_INTENTION|LOCK_PREDICATE|LOCK_PRDT_PAGE)&LOCK_TYPE_MASK
-# error
-#endif
-/* @} */
-
 /** Lock operation struct */
 struct lock_op_t{
 	dict_table_t*	table;	/*!< table to be locked */
diff --git a/storage/innobase/include/lock0priv.h b/storage/innobase/include/lock0priv.h
index 1c297a5b945..d9377e97d40 100644
--- a/storage/innobase/include/lock0priv.h
+++ b/storage/innobase/include/lock0priv.h
@@ -42,19 +42,6 @@ those functions in lock/ */
 #define UINT32_MAX             (4294967295U)
 #endif
 
-/** A table lock */
-struct lock_table_t {
-	dict_table_t*	table;		/*!< database table in dictionary
-					cache */
-	UT_LIST_NODE_T(lock_t)
-			locks;		/*!< list of locks on the same
-					table */
-	/** Print the table lock into the given output stream
-	@param[in,out]	out	the output stream
-	@return the given output stream. */
-	std::ostream& print(std::ostream& out) const;
-};
-
 /** Print the table lock into the given output stream
 @param[in,out]	out	the output stream
 @return the given output stream. */
@@ -77,131 +64,11 @@ operator<<(std::ostream& out, const lock_table_t& lock)
 	return(lock.print(out));
 }
 
-/** Record lock for a page */
-struct lock_rec_t {
-	ib_uint32_t	space;		/*!< space id */
-	ib_uint32_t	page_no;	/*!< page number */
-	ib_uint32_t	n_bits;		/*!< number of bits in the lock
-					bitmap; NOTE: the lock bitmap is
-					placed immediately after the
-					lock struct */
-
-	/** Print the record lock into the given output stream
-	@param[in,out]	out	the output stream
-	@return the given output stream. */
-	std::ostream& print(std::ostream& out) const;
-};
-
-/** Print the record lock into the given output stream
-@param[in,out]	out	the output stream
-@return the given output stream. */
-inline
-std::ostream& lock_rec_t::print(std::ostream& out) const
-{
-	out << "[lock_rec_t: space=" << space << ", page_no=" << page_no
-		<< ", n_bits=" << n_bits << "]";
-	return(out);
-}
-
-inline
-std::ostream&
-operator<<(std::ostream& out, const lock_rec_t& lock)
-{
-	return(lock.print(out));
-}
-
-/** Lock struct; protected by lock_sys->mutex */
-struct lock_t {
-	trx_t*		trx;		/*!< transaction owning the
-					lock */
-	UT_LIST_NODE_T(lock_t)
-			trx_locks;	/*!< list of the locks of the
-					transaction */
-
-	dict_index_t*	index;		/*!< index for a record lock */
-
-	lock_t*		hash;		/*!< hash chain node for a record
-					lock. The link node in a singly linked
-					list, used during hashing. */
-
-	/* Statistics for how long lock has been held and time
-	how long this lock had to be waited before it was granted */
-	time_t		requested_time; /*!< Lock request time */
-	ulint		wait_time;	/*!< Time waited this lock or 0 */
-
-	union {
-		lock_table_t	tab_lock;/*!< table lock */
-		lock_rec_t	rec_lock;/*!< record lock */
-	} un_member;			/*!< lock details */
-
-	ib_uint32_t	type_mode;	/*!< lock type, mode, LOCK_GAP or
-					LOCK_REC_NOT_GAP,
-					LOCK_INSERT_INTENTION,
-					wait flag, ORed */
-
-	/** Determine if the lock object is a record lock.
-	@return true if record lock, false otherwise. */
-	bool is_record_lock() const
-	{
-		return(type() == LOCK_REC);
-	}
-
-	bool is_waiting() const
-	{
-		return(type_mode & LOCK_WAIT);
-	}
-
-	bool is_gap() const
-	{
-		return(type_mode & LOCK_GAP);
-	}
-
-	bool is_record_not_gap() const
-	{
-		return(type_mode & LOCK_REC_NOT_GAP);
-	}
-
-	bool is_insert_intention() const
-	{
-		return(type_mode & LOCK_INSERT_INTENTION);
-	}
-
-	ulint type() const {
-		return(type_mode & LOCK_TYPE_MASK);
-	}
-
-	enum lock_mode mode() const
-	{
-		return(static_cast<enum lock_mode>(type_mode & LOCK_MODE_MASK));
-	}
-
-	/** Print the lock object into the given output stream.
-	@param[in,out]	out	the output stream
-	@return the given output stream. */
-	std::ostream& print(std::ostream& out) const;
-
-	/** Convert the member 'type_mode' into a human readable string.
-	@return human readable string */
-	std::string type_mode_string() const;
-
-	const char* type_string() const
-	{
-		switch (type_mode & LOCK_TYPE_MASK) {
-		case LOCK_REC:
-			return("LOCK_REC");
-		case LOCK_TABLE:
-			return("LOCK_TABLE");
-		default:
-			ut_error;
-		}
-	}
-};
-
 /** Convert the member 'type_mode' into a human readable string.
 @return human readable string */
 inline
 std::string
-lock_t::type_mode_string() const
+ib_lock_t::type_mode_string() const
 {
 	std::ostringstream sout;
 	sout << type_string();
@@ -227,7 +94,7 @@ lock_t::type_mode_string() const
 
 inline
 std::ostream&
-lock_t::print(std::ostream& out) const
+ib_lock_t::print(std::ostream& out) const
 {
 	out << "[lock_t: type_mode=" << type_mode << "("
 		<< type_mode_string() << ")";
@@ -244,7 +111,7 @@ lock_t::print(std::ostream& out) const
 
 inline
 std::ostream&
-operator<<(std::ostream& out, const lock_t& lock)
+operator<<(std::ostream& out, const ib_lock_t& lock)
 {
 	return(lock.print(out));
 }
diff --git a/storage/innobase/include/lock0types.h b/storage/innobase/include/lock0types.h
index d08eaabfb1e..1a03d1d0297 100644
--- a/storage/innobase/include/lock0types.h
+++ b/storage/innobase/include/lock0types.h
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -73,6 +74,195 @@ const char* lock_mode_string(enum lock_mode mode)
 	}
 }
 
-typedef UT_LIST_BASE_NODE_T(lock_t) trx_lock_list_t;
+/** A table lock */
+struct lock_table_t {
+	dict_table_t*	table;		/*!< database table in dictionary
+					cache */
+	UT_LIST_NODE_T(ib_lock_t)
+			locks;		/*!< list of locks on the same
+					table */
+	/** Print the table lock into the given output stream
+	@param[in,out]	out	the output stream
+	@return the given output stream. */
+	std::ostream& print(std::ostream& out) const;
+};
+
+/** Record lock for a page */
+struct lock_rec_t {
+	ib_uint32_t	space;		/*!< space id */
+	ib_uint32_t	page_no;	/*!< page number */
+	ib_uint32_t	n_bits;		/*!< number of bits in the lock
+					bitmap; NOTE: the lock bitmap is
+					placed immediately after the
+					lock struct */
+
+	/** Print the record lock into the given output stream
+	@param[in,out]	out	the output stream
+	@return the given output stream. */
+	std::ostream& print(std::ostream& out) const;
+};
+
+/** Print the record lock into the given output stream
+@param[in,out]	out	the output stream
+@return the given output stream. */
+inline
+std::ostream& lock_rec_t::print(std::ostream& out) const
+{
+	out << "[lock_rec_t: space=" << space << ", page_no=" << page_no
+		<< ", n_bits=" << n_bits << "]";
+	return(out);
+}
+
+inline
+std::ostream&
+operator<<(std::ostream& out, const lock_rec_t& lock)
+{
+	return(lock.print(out));
+}
+
+#define LOCK_MODE_MASK	0xFUL	/*!< mask used to extract mode from the
+				type_mode field in a lock */
+/** Lock types */
+/* @{ */
+#define LOCK_TABLE	16U	/*!< table lock */
+#define	LOCK_REC	32U	/*!< record lock */
+#define LOCK_TYPE_MASK	0xF0UL	/*!< mask used to extract lock type from the
+				type_mode field in a lock */
+#if LOCK_MODE_MASK & LOCK_TYPE_MASK
+# error "LOCK_MODE_MASK & LOCK_TYPE_MASK"
+#endif
+
+#define LOCK_WAIT	256U	/*!< Waiting lock flag; when set, it
+				means that the lock has not yet been
+				granted, it is just waiting for its
+				turn in the wait queue */
+/* Precise modes */
+#define LOCK_ORDINARY	0	/*!< this flag denotes an ordinary
+				next-key lock in contrast to LOCK_GAP
+				or LOCK_REC_NOT_GAP */
+#define LOCK_GAP	512U	/*!< when this bit is set, it means that the
+				lock holds only on the gap before the record;
+				for instance, an x-lock on the gap does not
+				give permission to modify the record on which
+				the bit is set; locks of this type are created
+				when records are removed from the index chain
+				of records */
+#define LOCK_REC_NOT_GAP 1024U	/*!< this bit means that the lock is only on
+				the index record and does NOT block inserts
+				to the gap before the index record; this is
+				used in the case when we retrieve a record
+				with a unique key, and is also used in
+				locking plain SELECTs (not part of UPDATE
+				or DELETE) when the user has set the READ
+				COMMITTED isolation level */
+#define LOCK_INSERT_INTENTION 2048U/*!< this bit is set when we place a waiting
+				gap type record lock request in order to let
+				an insert of an index record to wait until
+				there are no conflicting locks by other
+				transactions on the gap; note that this flag
+				remains set when the waiting lock is granted,
+				or if the lock is inherited to a neighboring
+				record */
+#define LOCK_PREDICATE	8192U	/*!< Predicate lock */
+#define LOCK_PRDT_PAGE	16384U	/*!< Page lock */
+
+
+#if (LOCK_WAIT|LOCK_GAP|LOCK_REC_NOT_GAP|LOCK_INSERT_INTENTION|LOCK_PREDICATE|LOCK_PRDT_PAGE)&LOCK_MODE_MASK
+# error
+#endif
+#if (LOCK_WAIT|LOCK_GAP|LOCK_REC_NOT_GAP|LOCK_INSERT_INTENTION|LOCK_PREDICATE|LOCK_PRDT_PAGE)&LOCK_TYPE_MASK
+# error
+#endif
+/* @} */
+
+/** Lock struct; protected by lock_sys->mutex */
+struct ib_lock_t
+{
+	trx_t*		trx;		/*!< transaction owning the
+					lock */
+	UT_LIST_NODE_T(ib_lock_t)
+			trx_locks;	/*!< list of the locks of the
+					transaction */
+
+	dict_index_t*	index;		/*!< index for a record lock */
+
+	ib_lock_t*	hash;		/*!< hash chain node for a record
+					lock. The link node in a singly linked
+					list, used during hashing. */
+
+	/* Statistics for how long lock has been held and time
+	how long this lock had to be waited before it was granted */
+	time_t		requested_time; /*!< Lock request time */
+	ulint		wait_time;	/*!< Time waited this lock or 0 */
+
+	union {
+		lock_table_t	tab_lock;/*!< table lock */
+		lock_rec_t	rec_lock;/*!< record lock */
+	} un_member;			/*!< lock details */
+
+	ib_uint32_t	type_mode;	/*!< lock type, mode, LOCK_GAP or
+					LOCK_REC_NOT_GAP,
+					LOCK_INSERT_INTENTION,
+					wait flag, ORed */
+
+	/** Determine if the lock object is a record lock.
+	@return true if record lock, false otherwise. */
+	bool is_record_lock() const
+	{
+		return(type() == LOCK_REC);
+	}
+
+	bool is_waiting() const
+	{
+		return(type_mode & LOCK_WAIT);
+	}
+
+	bool is_gap() const
+	{
+		return(type_mode & LOCK_GAP);
+	}
+
+	bool is_record_not_gap() const
+	{
+		return(type_mode & LOCK_REC_NOT_GAP);
+	}
+
+	bool is_insert_intention() const
+	{
+		return(type_mode & LOCK_INSERT_INTENTION);
+	}
+
+	ulint type() const {
+		return(type_mode & LOCK_TYPE_MASK);
+	}
+
+	enum lock_mode mode() const
+	{
+		return(static_cast<enum lock_mode>(type_mode & LOCK_MODE_MASK));
+	}
+
+	/** Print the lock object into the given output stream.
+	@param[in,out]	out	the output stream
+	@return the given output stream. */
+	std::ostream& print(std::ostream& out) const;
+
+	/** Convert the member 'type_mode' into a human readable string.
+	@return human readable string */
+	std::string type_mode_string() const;
+
+	const char* type_string() const
+	{
+		switch (type_mode & LOCK_TYPE_MASK) {
+		case LOCK_REC:
+			return("LOCK_REC");
+		case LOCK_TABLE:
+			return("LOCK_TABLE");
+		default:
+			ut_error;
+		}
+	}
+};
+
+typedef UT_LIST_BASE_NODE_T(ib_lock_t) trx_lock_list_t;
 
 #endif /* lock0types_h */
diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h
index 6ed69e6c5c4..adfe4f41955 100644
--- a/storage/innobase/include/trx0trx.h
+++ b/storage/innobase/include/trx0trx.h
@@ -704,13 +704,19 @@ struct trx_lock_t {
 					only be modified by the thread that is
 					serving the running transaction. */
 
-	lock_pool_t	rec_pool;	/*!< Pre-allocated record locks */
+	/** Pre-allocated record locks */
+	struct {
+		ib_lock_t lock; byte pad[256];
+	} rec_pool[8];
 
-	lock_pool_t	table_pool;	/*!< Pre-allocated table locks */
+	/** Pre-allocated table locks */
+	ib_lock_t	table_pool[8];
 
-	ulint		rec_cached;	/*!< Next free rec lock in pool */
+	/** Next available rec_pool[] entry */
+	unsigned	rec_cached;
 
-	ulint		table_cached;	/*!< Next free table lock in pool */
+	/** Next available table_pool[] entry */
+	unsigned	table_cached;
 
 	mem_heap_t*	lock_heap;	/*!< memory heap for trx_locks;
 					protected by lock_sys->mutex */
diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc
index 61f62d1e054..18e4c164860 100644
--- a/storage/innobase/lock/lock0lock.cc
+++ b/storage/innobase/lock/lock0lock.cc
@@ -59,18 +59,6 @@ ulong innodb_lock_schedule_algorithm;
 /** The value of innodb_deadlock_detect */
 my_bool	innobase_deadlock_detect;
 
-/** Total number of cached record locks */
-static const ulint	REC_LOCK_CACHE = 8;
-
-/** Maximum record lock size in bytes */
-static const ulint	REC_LOCK_SIZE = sizeof(ib_lock_t) + 256;
-
-/** Total number of cached table locks */
-static const ulint	TABLE_LOCK_CACHE = 8;
-
-/** Size in bytes, of the table lock instance */
-static const ulint	TABLE_LOCK_SIZE = sizeof(ib_lock_t);
-
 /*********************************************************************//**
 Checks if a waiting record lock request still has to wait in a queue.
 @return lock that is causing the wait */
@@ -1469,13 +1457,13 @@ lock_rec_create_low(
 		}
 	}
 
- 	if (trx->lock.rec_cached >= trx->lock.rec_pool.size()
-	    || sizeof *lock + n_bytes > REC_LOCK_SIZE) {
+	if (trx->lock.rec_cached >= UT_ARR_SIZE(trx->lock.rec_pool)
+	    || sizeof *lock + n_bytes > sizeof *trx->lock.rec_pool) {
 		lock = static_cast<lock_t*>(
 			mem_heap_alloc(trx->lock.lock_heap,
 				       sizeof *lock + n_bytes));
 	} else {
-		lock = trx->lock.rec_pool[trx->lock.rec_cached++];
+		lock = &trx->lock.rec_pool[trx->lock.rec_cached++].lock;
 	}
 
 	lock->trx = trx;
@@ -3653,8 +3641,9 @@ lock_table_create(
 
 		ib_vector_push(trx->autoinc_locks, &lock);
 
-	} else if (trx->lock.table_cached < trx->lock.table_pool.size()) {
-		lock = trx->lock.table_pool[trx->lock.table_cached++];
+	} else if (trx->lock.table_cached
+		   < UT_ARR_SIZE(trx->lock.table_pool)) {
+		lock = &trx->lock.table_pool[trx->lock.table_cached++];
 	} else {
 
 		lock = static_cast<lock_t*>(
@@ -7678,33 +7667,6 @@ DeadlockChecker::check_and_resolve(const lock_t* lock, trx_t* trx)
 	return(victim_trx);
 }
 
-/**
-Allocate cached locks for the transaction.
-@param trx		allocate cached record locks for this transaction */
-void
-lock_trx_alloc_locks(trx_t* trx)
-{
-	ulint	sz = REC_LOCK_SIZE * REC_LOCK_CACHE;
-	byte*	ptr = reinterpret_cast<byte*>(ut_malloc_nokey(sz));
-
-	/* We allocate one big chunk and then distribute it among
-	the rest of the elements. The allocated chunk pointer is always
-	at index 0. */
-
-	for (ulint i = 0; i < REC_LOCK_CACHE; ++i, ptr += REC_LOCK_SIZE) {
-		trx->lock.rec_pool.push_back(
-			reinterpret_cast<ib_lock_t*>(ptr));
-	}
-
-	sz = TABLE_LOCK_SIZE * TABLE_LOCK_CACHE;
-	ptr = reinterpret_cast<byte*>(ut_malloc_nokey(sz));
-
-	for (ulint i = 0; i < TABLE_LOCK_CACHE; ++i, ptr += TABLE_LOCK_SIZE) {
-		trx->lock.table_pool.push_back(
-			reinterpret_cast<ib_lock_t*>(ptr));
-	}
-
-}
 /*************************************************************//**
 Updates the lock table when a page is split and merged to
 two pages. */
diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc
index 7ebf41b3143..ae2611020de 100644
--- a/storage/innobase/trx/trx0trx.cc
+++ b/storage/innobase/trx/trx0trx.cc
@@ -190,10 +190,6 @@ struct TrxFactory {
 		the constructors of the trx_t members. */
 		new(&trx->mod_tables) trx_mod_tables_t();
 
-		new(&trx->lock.rec_pool) lock_pool_t();
-
-		new(&trx->lock.table_pool) lock_pool_t();
-
 		new(&trx->lock.table_locks) lock_pool_t();
 
 		trx_init(trx);
@@ -216,8 +212,6 @@ struct TrxFactory {
 
 		mutex_create(LATCH_ID_TRX, &trx->mutex);
 		mutex_create(LATCH_ID_TRX_UNDO, &trx->undo_mutex);
-
-		lock_trx_alloc_locks(trx);
 	}
 
 	/** Release resources held by the transaction object.
@@ -249,26 +243,6 @@ struct TrxFactory {
 
 		ut_ad(trx->read_view == NULL);
 
-		if (!trx->lock.rec_pool.empty()) {
-
-			/* See lock_trx_alloc_locks() why we only free
-			the first element. */
-
-			ut_free(trx->lock.rec_pool[0]);
-		}
-
-		if (!trx->lock.table_pool.empty()) {
-
-			/* See lock_trx_alloc_locks() why we only free
-			the first element. */
-
-			ut_free(trx->lock.table_pool[0]);
-		}
-
-		trx->lock.rec_pool.~lock_pool_t();
-
-		trx->lock.table_pool.~lock_pool_t();
-
 		trx->lock.table_locks.~lock_pool_t();
 	}
 
@@ -407,7 +381,12 @@ trx_create_low()
 
 	/* Should have been either just initialized or .clear()ed by
 	trx_free(). */
-	ut_a(trx->mod_tables.size() == 0);
+	ut_ad(trx->mod_tables.empty());
+	ut_ad(trx->lock.table_locks.empty());
+	ut_ad(UT_LIST_GET_LEN(trx->lock.trx_locks) == 0);
+	ut_ad(trx->lock.n_rec_locks == 0);
+	ut_ad(trx->lock.table_cached == 0);
+	ut_ad(trx->lock.rec_cached == 0);
 
 #ifdef WITH_WSREP
 	trx->wsrep_event = NULL;

From 658350657005569f11bd6f9c97753a6cf24427f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Mon, 13 Aug 2018 12:34:12 +0300
Subject: [PATCH 160/164] Rename lock_pool_t to lock_list

Also, replace reverse iteration with forward iteration.

lock_table_has(): Remove a redundant condition.
---
 storage/innobase/include/lock0priv.ic | 12 +++-------
 storage/innobase/include/trx0trx.h    |  4 ++--
 storage/innobase/lock/lock0lock.cc    | 32 ++++++++-------------------
 storage/innobase/trx/trx0trx.cc       |  4 ++--
 4 files changed, 16 insertions(+), 36 deletions(-)

diff --git a/storage/innobase/include/lock0priv.ic b/storage/innobase/include/lock0priv.ic
index 150a80b7be4..b5be9076644 100644
--- a/storage/innobase/include/lock0priv.ic
+++ b/storage/innobase/include/lock0priv.ic
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2007, 2014, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -388,17 +389,10 @@ lock_table_has(
 	const dict_table_t*	table,	/*!< in: table */
 	lock_mode		in_mode)/*!< in: lock mode */
 {
-	if (trx->lock.table_locks.empty()) {
-		return(NULL);
-	}
-
-	typedef lock_pool_t::const_reverse_iterator iterator;
-
-	iterator	end = trx->lock.table_locks.rend();
-
 	/* Look for stronger locks the same trx already has on the table */
 
-	for (iterator it = trx->lock.table_locks.rbegin(); it != end; ++it) {
+	for (lock_list::const_iterator it = trx->lock.table_locks.begin(),
+             end = trx->lock.table_locks.end(); it != end; ++it) {
 
 		const lock_t*	lock = *it;
 
diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h
index adfe4f41955..01670119381 100644
--- a/storage/innobase/include/trx0trx.h
+++ b/storage/innobase/include/trx0trx.h
@@ -642,7 +642,7 @@ The tranasction must be in the mysql_trx_list. */
 # define assert_trx_nonlocking_or_in_list(trx) ((void)0)
 #endif /* UNIV_DEBUG */
 
-typedef std::vector<ib_lock_t*, ut_allocator<ib_lock_t*> >	lock_pool_t;
+typedef std::vector<ib_lock_t*, ut_allocator<ib_lock_t*> >	lock_list;
 
 /*******************************************************************//**
 Latching protocol for trx_lock_t::que_state.  trx_lock_t::que_state
@@ -726,7 +726,7 @@ struct trx_lock_t {
 					and lock_sys->mutex; removals are
 					protected by lock_sys->mutex */
 
-	lock_pool_t	table_locks;	/*!< All table locks requested by this
+	lock_list	table_locks;	/*!< All table locks requested by this
 					transaction, including AUTOINC locks */
 
 	bool		cancel;		/*!< true if the transaction is being
diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc
index 18e4c164860..90cae86073a 100644
--- a/storage/innobase/lock/lock0lock.cc
+++ b/storage/innobase/lock/lock0lock.cc
@@ -4493,24 +4493,15 @@ lock_trx_table_locks_remove(
 		ut_ad(trx_mutex_own(trx));
 	}
 
-	typedef lock_pool_t::reverse_iterator iterator;
-
-	iterator	end = trx->lock.table_locks.rend();
-
-	for (iterator it = trx->lock.table_locks.rbegin(); it != end; ++it) {
-
+	for (lock_list::iterator it = trx->lock.table_locks.begin(),
+             end = trx->lock.table_locks.end(); it != end; ++it) {
 		const lock_t*	lock = *it;
 
-		if (lock == NULL) {
-			continue;
-		}
-
-		ut_a(trx == lock->trx);
-		ut_a(lock_get_type_low(lock) & LOCK_TABLE);
-		ut_a(lock->un_member.tab_lock.table != NULL);
+		ut_ad(!lock || trx == lock->trx);
+		ut_ad(!lock || lock_get_type_low(lock) & LOCK_TABLE);
+		ut_ad(!lock || lock->un_member.tab_lock.table);
 
 		if (lock == lock_to_remove) {
-
 			*it = NULL;
 
 			if (!trx->lock.cancel) {
@@ -5370,11 +5361,8 @@ lock_trx_table_locks_find(
 
 	trx_mutex_enter(trx);
 
-	typedef lock_pool_t::const_reverse_iterator iterator;
-
-	iterator	end = trx->lock.table_locks.rend();
-
-	for (iterator it = trx->lock.table_locks.rbegin(); it != end; ++it) {
+	for (lock_list::const_iterator it = trx->lock.table_locks.begin(),
+             end = trx->lock.table_locks.end(); it != end; ++it) {
 
 		const lock_t*	lock = *it;
 
@@ -7108,10 +7096,8 @@ lock_trx_has_sys_table_locks(
 
 	lock_mutex_enter();
 
-	typedef lock_pool_t::const_reverse_iterator iterator;
-
-	iterator	end = trx->lock.table_locks.rend();
-	iterator	it = trx->lock.table_locks.rbegin();
+	const lock_list::const_iterator end = trx->lock.table_locks.end();
+	lock_list::const_iterator it = trx->lock.table_locks.begin();
 
 	/* Find a valid mode. Note: ib_vector_size() can be 0. */
 
diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc
index ae2611020de..b79c9403c94 100644
--- a/storage/innobase/trx/trx0trx.cc
+++ b/storage/innobase/trx/trx0trx.cc
@@ -190,7 +190,7 @@ struct TrxFactory {
 		the constructors of the trx_t members. */
 		new(&trx->mod_tables) trx_mod_tables_t();
 
-		new(&trx->lock.table_locks) lock_pool_t();
+		new(&trx->lock.table_locks) lock_list();
 
 		trx_init(trx);
 
@@ -243,7 +243,7 @@ struct TrxFactory {
 
 		ut_ad(trx->read_view == NULL);
 
-		trx->lock.table_locks.~lock_pool_t();
+		trx->lock.table_locks.~lock_list();
 	}
 
 	/** Enforce any invariants here, this is called before the transaction

From fa2a74e08d00bcf8ddb1d98dba767e9bd1ed802d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Mon, 13 Aug 2018 13:32:05 +0300
Subject: [PATCH 161/164] MDEV-16136: Prevent wrong reuse in trx_reference()

If trx_free() and trx_create_low() were called while a call to
trx_reference() was pending, we could get a reference to a wrong
transaction object.

trx_reference(): Return NULL if the trx->id no longer matches.

lock_trx_release_locks(): Assign trx->id = 0, so that trx_reference()
will not return a reference to this object.

trx_cleanup_at_db_startup(): Assign trx->id = 0.

assert_trx_is_free(): Assert !trx->n_ref. Assert trx->id == 0,
now that it will be cleared as part of a transaction commit.
---
 storage/innobase/include/trx0sys.ic |  9 +++---
 storage/innobase/include/trx0trx.h  | 43 +++++++++++++++++++----------
 storage/innobase/include/trx0trx.ic | 29 -------------------
 storage/innobase/lock/lock0lock.cc  |  2 ++
 storage/innobase/trx/trx0trx.cc     | 10 +++----
 5 files changed, 40 insertions(+), 53 deletions(-)

diff --git a/storage/innobase/include/trx0sys.ic b/storage/innobase/include/trx0sys.ic
index af6fb63d9b4..861800ef40e 100644
--- a/storage/innobase/include/trx0sys.ic
+++ b/storage/innobase/include/trx0sys.ic
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -360,14 +361,14 @@ trx_rw_is_active(
 	bool		do_ref_count)	/*!< in: if true then increment the
 					trx_t::n_ref_count */
 {
-	trx_t*		trx;
+	ut_ad(trx_id);
 
 	trx_sys_mutex_enter();
 
-	trx = trx_rw_is_active_low(trx_id, corrupt);
+	trx_t* trx = trx_rw_is_active_low(trx_id, corrupt);
 
-	if (trx != 0) {
-		trx = trx_reference(trx, do_ref_count);
+	if (trx) {
+		trx = trx_reference(do_ref_count ? trx_id : 0, trx);
 	}
 
 	trx_sys_mutex_exit();
diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h
index 01670119381..917222477b1 100644
--- a/storage/innobase/include/trx0trx.h
+++ b/storage/innobase/include/trx0trx.h
@@ -518,19 +518,6 @@ void
 trx_set_rw_mode(
 	trx_t*		trx);
 
-/**
-Increase the reference count. If the transaction is in state
-TRX_STATE_COMMITTED_IN_MEMORY then the transaction is considered
-committed and the reference count is not incremented.
-@param trx Transaction that is being referenced
-@param do_ref_count Increment the reference iff this is true
-@return transaction instance if it is not committed */
-UNIV_INLINE
-trx_t*
-trx_reference(
-	trx_t*		trx,
-	bool		do_ref_count);
-
 /**
 Release the transaction. Decrease the reference count.
 @param trx Transaction that is being released */
@@ -600,7 +587,9 @@ Check transaction state */
 @param t transaction handle */
 #define	assert_trx_is_free(t)	do {					\
 	ut_ad(trx_state_eq((t), TRX_STATE_NOT_STARTED));		\
-	ut_ad(!trx->has_logged());					\
+	ut_ad(!(t)->id);						\
+	ut_ad(!(t)->has_logged());					\
+	ut_ad(!(t)->n_ref);						\
 	ut_ad(!MVCC::is_view_active((t)->read_view));			\
 	ut_ad((t)->lock.wait_thr == NULL);				\
 	ut_ad(UT_LIST_GET_LEN((t)->lock.trx_locks) == 0);		\
@@ -1277,6 +1266,32 @@ struct commit_node_t{
 	mutex_exit(&t->mutex);			\
 } while (0)
 
+/**
+Increase the reference count. If the transaction is in state
+TRX_STATE_COMMITTED_IN_MEMORY then the transaction is considered
+committed and the reference count is not incremented.
+@param id the transaction ID; 0 if not to increment the reference count
+@param trx Transaction that is being referenced
+@return trx
+@retval	NULL	if the transaction is no longer active */
+inline trx_t* trx_reference(trx_id_t id, trx_t* trx)
+{
+	trx_mutex_enter(trx);
+
+	if (trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY)) {
+		trx = NULL;
+	} else if (!id) {
+	} else if (trx->id != id) {
+		trx = NULL;
+	} else {
+		ut_ad(trx->n_ref >= 0);
+		++trx->n_ref;
+	}
+
+	trx_mutex_exit(trx);
+	return(trx);
+}
+
 #include "trx0trx.ic"
 
 #endif
diff --git a/storage/innobase/include/trx0trx.ic b/storage/innobase/include/trx0trx.ic
index 7721e28bfb6..dd42c8b8368 100644
--- a/storage/innobase/include/trx0trx.ic
+++ b/storage/innobase/include/trx0trx.ic
@@ -210,35 +210,6 @@ ok:
 	trx->dict_operation = op;
 }
 
-/**
-Increase the reference count. If the transaction is in state
-TRX_STATE_COMMITTED_IN_MEMORY then the transaction is considered
-committed and the reference count is not incremented.
-@param trx Transaction that is being referenced
-@param do_ref_count Increment the reference iff this is true
-@return transaction instance if it is not committed */
-UNIV_INLINE
-trx_t*
-trx_reference(
-	trx_t*		trx,
-	bool		do_ref_count)
-{
-	trx_mutex_enter(trx);
-
-	if (trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY)) {
-		trx_mutex_exit(trx);
-		trx = NULL;
-	} else if (do_ref_count) {
-		ut_ad(trx->n_ref >= 0);
-		++trx->n_ref;
-		trx_mutex_exit(trx);
-	} else {
-		trx_mutex_exit(trx);
-	}
-
-	return(trx);
-}
-
 /**
 Release the transaction. Decrease the reference count.
 @param trx Transaction that is being released */
diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc
index 90cae86073a..b82f323d8f9 100644
--- a/storage/innobase/lock/lock0lock.cc
+++ b/storage/innobase/lock/lock0lock.cc
@@ -6892,6 +6892,8 @@ lock_trx_release_locks(
 	the is_recovered flag. */
 
 	trx->is_recovered = false;
+	/* Ensure that trx_reference() will not find this transaction. */
+	trx->id = 0;
 
 	trx_mutex_exit(trx);
 
diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc
index b79c9403c94..e4d9c0800c3 100644
--- a/storage/innobase/trx/trx0trx.cc
+++ b/storage/innobase/trx/trx0trx.cc
@@ -111,8 +111,6 @@ trx_init(
 /*=====*/
 	trx_t*	trx)
 {
-	trx->id = 0;
-
 	trx->no = TRX_ID_MAX;
 
 	trx->state = TRX_STATE_NOT_STARTED;
@@ -1223,10 +1221,7 @@ trx_start_low(
 		ut_ad(trx_sys_validate_trx_list());
 
 		trx_sys_mutex_exit();
-
 	} else {
-		trx->id = 0;
-
 		if (!trx_is_autocommit_non_locking(trx)) {
 
 			/* If this is a read-only transaction that is writing
@@ -1668,7 +1663,10 @@ trx_commit_in_memory(
 			trx_erase_lists(trx, serialised);
 		}
 
+		/* trx->id will be cleared in lock_trx_release_locks(trx). */
+		ut_ad(trx->read_only || !trx->rsegs.m_redo.rseg || trx->id);
 		lock_trx_release_locks(trx);
+		ut_ad(trx->id == 0);
 
 		/* Remove the transaction from the list of active
 		transactions now that it no longer holds any user locks. */
@@ -1685,7 +1683,6 @@ trx_commit_in_memory(
 			}
 
 		} else {
-			ut_ad(trx->id > 0);
 			MONITOR_INC(MONITOR_TRX_RW_COMMIT);
 		}
 	}
@@ -1956,6 +1953,7 @@ trx_cleanup_at_db_startup(
 	ut_ad(!trx->in_rw_trx_list);
 	ut_ad(!trx->in_mysql_trx_list);
 	DBUG_LOG("trx", "Cleanup at startup: " << trx);
+	trx->id = 0;
 	trx->state = TRX_STATE_NOT_STARTED;
 }
 

From a0ce321627a13658d5e8a37d7c3dfb0ef26ae0ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Wed, 15 Aug 2018 18:45:15 +0300
Subject: [PATCH 162/164] MDEV-16136 Various ASAN failures after MDEV-15030

The Pool poisoning that was introduced in MDEV-15030 introduced
race conditions in AddressSanitizer builds, because concurrent
poisoning and unpoisoning were not prevented by any synchronization
primitive.

Pool::get(): Protect the unpoisoning by m_lock_strategy.

Pool::mem_free(): Protect the poisoning by m_lock_strategy.

Pool::putl(): Renamed from put(), because now the caller is
responsible for invoking m_lock_strategy.
---
 storage/innobase/include/ut0pool.h | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/storage/innobase/include/ut0pool.h b/storage/innobase/include/ut0pool.h
index 5b5afe7cd80..c0237158ce5 100644
--- a/storage/innobase/include/ut0pool.h
+++ b/storage/innobase/include/ut0pool.h
@@ -125,8 +125,7 @@ struct Pool {
 			elem = NULL;
 		}
 
-		m_lock_strategy.exit();
-
+#if defined HAVE_valgrind || defined __SANITIZE_ADDRESS__
 		if (elem) {
 			/* Unpoison the memory for AddressSanitizer */
 			MEM_UNDEFINED(&elem->m_type, sizeof elem->m_type);
@@ -135,10 +134,11 @@ struct Pool {
 			actually initialized; we checked that by
 			UNIV_MEM_ASSERT_RW() in mem_free() below. */
 			UNIV_MEM_VALID(&elem->m_type, sizeof elem->m_type);
-			return &elem->m_type;
 		}
+#endif
 
-		return NULL;
+		m_lock_strategy.exit();
+		return elem ? &elem->m_type : NULL;
 	}
 
 	/** Add the object to the pool.
@@ -151,8 +151,12 @@ struct Pool {
 		elem = reinterpret_cast<Element*>(p - sizeof(*elem));
 		UNIV_MEM_ASSERT_RW(&elem->m_type, sizeof elem->m_type);
 
-		elem->m_pool->put(elem);
+		elem->m_pool->m_lock_strategy.enter();
+
+		elem->m_pool->putl(elem);
 		MEM_NOACCESS(&elem->m_type, sizeof elem->m_type);
+
+		elem->m_pool->m_lock_strategy.exit();
 	}
 
 protected:
@@ -170,17 +174,13 @@ private:
 
 	/** Release the object to the free pool
 	@param elem element to free */
-	void put(Element* elem)
+	void putl(Element* elem)
 	{
-		m_lock_strategy.enter();
-
 		ut_ad(elem >= m_start && elem < m_last);
 
 		ut_ad(Factory::debug(&elem->m_type));
 
 		m_pqueue.push(elem);
-
-		m_lock_strategy.exit();
 	}
 
 	/** Initialise the elements.

From 05153a670de6eee0959ce564e11bb2f7ca820d42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Thu, 16 Aug 2018 06:35:28 +0300
Subject: [PATCH 163/164] Report all redo log corruption

Amend commit b853b4fd88b441f36eeb7eabdce79918ef10538a
that was reverted in commit 29150e23912ef6e1fe386f53b778c9a5551c4790.

recv_parse_log_recs(): Do check for corrupted redo log or file
system before checking for len==0, but only read *ptr if
it is not past the end of the buffer (end_ptr).

recv_parse_log_rec(): Report incorrect redo log type
in a consistent way with recv_parse_or_apply_log_rec_body().
This is a follow-up to commit f30c5af42e4e72a3a0d8d5fb7b9fc884a249f292.
---
 storage/innobase/log/log0recv.cc | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc
index 77e61606c1e..ee4d627261a 100644
--- a/storage/innobase/log/log0recv.cc
+++ b/storage/innobase/log/log0recv.cc
@@ -2253,7 +2253,8 @@ recv_parse_log_rec(
 	case MLOG_MULTI_REC_END | MLOG_SINGLE_REC_FLAG:
 	case MLOG_DUMMY_RECORD | MLOG_SINGLE_REC_FLAG:
 	case MLOG_CHECKPOINT | MLOG_SINGLE_REC_FLAG:
-		ib::error() << "Incorrect log record type:" << *ptr;
+		ib::error() << "Incorrect log record type "
+			<< ib::hex(unsigned(*ptr));
 		recv_sys->found_corrupt_log = true;
 		return(0);
 	}
@@ -2422,10 +2423,6 @@ loop:
 		len = recv_parse_log_rec(&type, ptr, end_ptr, &space,
 					 &page_no, apply, &body);
 
-		if (len == 0) {
-			return(false);
-		}
-
 		if (recv_sys->found_corrupt_log) {
 			recv_report_corrupt_log(ptr, type, space, page_no);
 			return(true);
@@ -2435,6 +2432,10 @@ loop:
 			return(true);
 		}
 
+		if (len == 0) {
+			return(false);
+		}
+
 		new_recovered_lsn = recv_calc_lsn_on_data_add(old_lsn, len);
 
 		if (new_recovered_lsn > recv_sys->scanned_lsn) {
@@ -2558,13 +2559,10 @@ loop:
 				&type, ptr, end_ptr, &space, &page_no,
 				false, &body);
 
-			if (len == 0) {
-				return(false);
-			}
-
 			if (recv_sys->found_corrupt_log
 			    || type == MLOG_CHECKPOINT
-			    || (*ptr & MLOG_SINGLE_REC_FLAG)) {
+			    || (ptr != end_ptr
+				&& (*ptr & MLOG_SINGLE_REC_FLAG))) {
 				recv_sys->found_corrupt_log = true;
 				recv_report_corrupt_log(
 					ptr, type, space, page_no);
@@ -2575,6 +2573,10 @@ loop:
 				return(true);
 			}
 
+			if (len == 0) {
+				return(false);
+			}
+
 			recv_previous_parsed_rec_type = type;
 			recv_previous_parsed_rec_offset
 				= recv_sys->recovered_offset + total_len;

From 8716bb3b72b7f5fed69b6dde413c2138f6d175b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Thu, 16 Aug 2018 09:41:20 +0300
Subject: [PATCH 164/164] After-merge fix: Restore DECLARE_THREAD to fix
 Windows build

---
 extra/mariabackup/xtrabackup.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc
index 611a07707dd..c57fc274062 100644
--- a/extra/mariabackup/xtrabackup.cc
+++ b/extra/mariabackup/xtrabackup.cc
@@ -2685,7 +2685,7 @@ void backup_wait_for_lsn(lsn_t lsn) {
 
 extern lsn_t server_lsn_after_lock;
 
-static os_thread_ret_t log_copying_thread(void*)
+static os_thread_ret_t DECLARE_THREAD(log_copying_thread)(void*)
 {
 	/*
 	  Initialize mysys thread-specific memory so we can