MDEV-11196: Error:Run-Time Check Failure #2 - Stack around the variable 'key_buff'

was corrupted, server crashes in opt_sum_query

Extended keys feature disabled if the length of extended key is longer than MAX_KEY_LEN

mysql-test/r/innodb_ext_key.result

@@ -1133,5 +1133,86 @@ where index_date_updated= 10 and index_id < 800;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t2	range	index_date_updated	index_date_updated	13	NULL	#	Using index condition
 drop table t0,t1,t2;
+#
+# MDEV-11196: Error:Run-Time Check Failure #2 - Stack around the variable 'key_buff'
+# was corrupted, server crashes in opt_sum_query
+set @save_innodb_file_format= @@innodb_file_format;
+set @save_innodb_large_prefix= @@innodb_large_prefix;
+set global innodb_file_format = BARRACUDA;
+set global innodb_large_prefix = ON;
+CREATE TABLE t1 (
+pk INT,
+f1 VARCHAR(3),
+f2 VARCHAR(1024),
+PRIMARY KEY (pk),
+KEY(f2)
+) ENGINE=InnoDB CHARSET utf8 ROW_FORMAT= DYNAMIC;
+INSERT INTO t1 VALUES (1,'foo','abc'),(2,'bar','def');
+SELECT MAX(t2.pk) FROM t1 t2 INNER JOIN t1 t3 ON t2.f1 = t3.f1 WHERE t2.pk <= 4;
+MAX(t2.pk)
+2
+drop table t1;
+CREATE TABLE t1 (
+pk1 INT,
+pk2 INT,
+f1 VARCHAR(3),
+f2 VARCHAR(1021),
+PRIMARY KEY (pk1,pk2),
+KEY(f2)
+) ENGINE=InnoDB CHARSET utf8 ROW_FORMAT= DYNAMIC;
+INSERT INTO t1 VALUES (1,2,'2','abc'),(2,3,'3','def');
+explain format= json
+select * from t1 force index(f2)  where pk1 <= 5 and pk2 <=5 and f2 = 'abc' and f1 <= '3';
+EXPLAIN
+{
+  "query_block": {
+    "select_id": 1,
+    "table": {
+      "table_name": "t1",
+      "access_type": "range",
+      "possible_keys": ["f2"],
+      "key": "f2",
+      "key_length": "3070",
+      "used_key_parts": ["f2", "pk1"],
+      "rows": 1,
+      "filtered": 100,
+      "index_condition": "((t1.pk1 <= 5) and (t1.pk2 <= 5) and (t1.f2 = 'abc'))",
+      "attached_condition": "(t1.f1 <= '3')"
+    }
+  }
+}
+drop table t1;
+CREATE TABLE t1 (
+f2 INT,
+pk2 INT,
+f1 VARCHAR(3),
+pk1 VARCHAR(1000),
+PRIMARY KEY (pk1,pk2),
+KEY k1(pk1,f2)
+) ENGINE=InnoDB CHARSET utf8 ROW_FORMAT= DYNAMIC;
+INSERT INTO t1 VALUES (1,2,'2','abc'),(2,3,'3','def');
+explain format= json
+select * from t1 force index(k1)  where f2 <= 5 and pk2 <=5 and pk1 = 'abc' and f1 <= '3';
+EXPLAIN
+{
+  "query_block": {
+    "select_id": 1,
+    "table": {
+      "table_name": "t1",
+      "access_type": "range",
+      "possible_keys": ["k1"],
+      "key": "k1",
+      "key_length": "3011",
+      "used_key_parts": ["pk1", "f2", "pk2"],
+      "rows": 1,
+      "filtered": 100,
+      "index_condition": "((t1.f2 <= 5) and (t1.pk2 <= 5) and (t1.pk1 = 'abc'))",
+      "attached_condition": "(t1.f1 <= '3')"
+    }
+  }
+}
+drop table t1;
 set optimizer_switch=@save_ext_key_optimizer_switch;
+set global innodb_file_format = @save_innodb_file_format;
+set global innodb_large_prefix = @save_innodb_large_prefix;
 SET SESSION STORAGE_ENGINE=DEFAULT;

mysql-test/t/innodb_ext_key.test

@@ -778,5 +778,56 @@ where index_date_updated= 10 and index_id < 800;
 
 drop table t0,t1,t2;
 
+
+--echo #
+--echo # MDEV-11196: Error:Run-Time Check Failure #2 - Stack around the variable 'key_buff'
+--echo # was corrupted, server crashes in opt_sum_query
+
+set @save_innodb_file_format= @@innodb_file_format;
+set @save_innodb_large_prefix= @@innodb_large_prefix;
+set global innodb_file_format = BARRACUDA;
+set global innodb_large_prefix = ON;
+
+CREATE TABLE t1 (
+  pk INT,
+  f1 VARCHAR(3),
+  f2 VARCHAR(1024),
+  PRIMARY KEY (pk),
+  KEY(f2)
+) ENGINE=InnoDB CHARSET utf8 ROW_FORMAT= DYNAMIC;
+
+INSERT INTO t1 VALUES (1,'foo','abc'),(2,'bar','def');
+SELECT MAX(t2.pk) FROM t1 t2 INNER JOIN t1 t3 ON t2.f1 = t3.f1 WHERE t2.pk <= 4;
+drop table t1;
+
+CREATE TABLE t1 (
+  pk1 INT,
+  pk2 INT,
+  f1 VARCHAR(3),
+  f2 VARCHAR(1021),
+  PRIMARY KEY (pk1,pk2),
+  KEY(f2)
+) ENGINE=InnoDB CHARSET utf8 ROW_FORMAT= DYNAMIC;
+
+INSERT INTO t1 VALUES (1,2,'2','abc'),(2,3,'3','def');
+explain format= json
+select * from t1 force index(f2)  where pk1 <= 5 and pk2 <=5 and f2 = 'abc' and f1 <= '3';
+drop table t1;
+
+CREATE TABLE t1 (
+f2 INT,
+pk2 INT,
+f1 VARCHAR(3),
+pk1 VARCHAR(1000),
+PRIMARY KEY (pk1,pk2),
+KEY k1(pk1,f2)
+) ENGINE=InnoDB CHARSET utf8 ROW_FORMAT= DYNAMIC;
+INSERT INTO t1 VALUES (1,2,'2','abc'),(2,3,'3','def');
+explain format= json
+select * from t1 force index(k1)  where f2 <= 5 and pk2 <=5 and pk1 = 'abc' and f1 <= '3';
+drop table t1;
+
 set optimizer_switch=@save_ext_key_optimizer_switch;
+set global innodb_file_format = @save_innodb_file_format;
+set global innodb_large_prefix = @save_innodb_large_prefix;
 SET SESSION STORAGE_ENGINE=DEFAULT;

sql/table.cc

@@ -1721,6 +1721,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
     keyinfo= share->key_info;
     uint primary_key= my_strcasecmp(system_charset_info, share->keynames.type_names[0],
                                     primary_key_name) ? MAX_KEY : 0;
+    KEY* key_first_info;
 
     if (primary_key >= MAX_KEY && keyinfo->flags & HA_NOSAME)
     {
@@ -1800,11 +1801,16 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
                keyinfo->name_length+1);
       }
 
+      if (!key)
+        key_first_info= keyinfo;
+
       if (ext_key_parts > share->key_parts && key)
       {
         KEY_PART_INFO *new_key_part= (keyinfo-1)->key_part +
                                      (keyinfo-1)->ext_key_parts;
         uint add_keyparts_for_this_key= add_first_key_parts;
+        uint length_bytes= 0, len_null_byte= 0, ext_key_length= 0;
+        Field *field;
 
         /* 
           Do not extend the key that contains a component
@@ -1813,6 +1819,20 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
         for (i= 0; i < keyinfo->user_defined_key_parts; i++)
         {
           uint fieldnr= keyinfo->key_part[i].fieldnr;
+          field= share->field[keyinfo->key_part[i].fieldnr-1];
+
+          if (field->null_ptr)
+            len_null_byte= HA_KEY_NULL_LENGTH;
+
+          if (field->type() == MYSQL_TYPE_BLOB ||
+             field->real_type() == MYSQL_TYPE_VARCHAR ||
+             field->type() == MYSQL_TYPE_GEOMETRY)
+          {
+            length_bytes= HA_KEY_BLOB_LENGTH;
+          }
+
+          ext_key_length+= keyinfo->key_part[i].length + len_null_byte
+                            + length_bytes;
           if (share->field[fieldnr-1]->key_length() !=
               keyinfo->key_part[i].length)
 	  {
@@ -1821,6 +1841,23 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
           }
         }
 
+        if (add_keyparts_for_this_key)
+        {
+          for (i= 0; i < add_keyparts_for_this_key; i++)
+          {
+            uint pk_part_length= key_first_info->key_part[i].store_length;
+            if (keyinfo->ext_key_part_map & 1<<i)
+            {
+              if (ext_key_length + pk_part_length > MAX_KEY_LENGTH)
+              {
+                add_keyparts_for_this_key= i;
+                break;
+              }
+              ext_key_length+= pk_part_length;
+            }
+          }
+        }
+
         if (add_keyparts_for_this_key < (keyinfo->ext_key_parts -
                                         keyinfo->user_defined_key_parts))
 	{