MDEV-24726 Assertion on compressed varstring as key field in optimizer

temporary table

Compressed field cannot be part of a key by its nature: there is no
data to order, only the compressed data.

For optimizer temporary table we create uncompressed substitute.

In all other cases (MDEV-16808) we don't use key: add_keyuse() is
skipped by !field->compression_method() condition.

mysql-test/main/column_compression.result

@@ -2978,4 +2978,60 @@ SELECT GROUP_CONCAT( b, a ORDER BY 2 ) AS f FROM t1;
 f
 nc,mmmmmmmmmmd
 DROP TABLE t1;
+#
+# MDEV-24726 Assertion `0' failed in Field_varstring_compressed::key_cmp
+#
+# VARCHAR
+create table t1 (a varchar(8) compressed) character set utf8mb4;
+create algorithm=temptable view v1 as select * from t1;
+insert into t1 values ('foo'),('bar'),('foo');
+select * from v1 where a in (select a from t1);
+a
+foo
+foo
+bar
+drop view v1;
+drop table t1;
+create table t1 (f1 varchar(8)) charset=eucjpms collate=eucjpms_nopad_bin;
+insert into t1 values ('');
+create table t2 (f2 varchar(8) compressed) charset=eucjpms collate=eucjpms_nopad_bin;
+insert into t2 values ('a'),('b');
+select t1.* from t1 left join (select distinct f2 from t2) sq on sq.f2 = t1.f1;
+f1
+
+drop table t1, t2;
+# BLOB
+create table t1 (a text compressed) character set utf8mb4;
+create algorithm=temptable view v1 as select * from t1;
+insert into t1 values ('foo'),('bar'),('foo');
+select * from v1 where a in (select a from t1);
+a
+foo
+foo
+bar
+drop view v1;
+drop table t1;
+create table t1 (f1 text) charset=eucjpms collate=eucjpms_nopad_bin;
+insert into t1 values ('');
+create table t2 (f2 text compressed) charset=eucjpms collate=eucjpms_nopad_bin;
+insert into t2 values ('a'),('b');
+select t1.* from t1 left join (select distinct f2 from t2) sq on sq.f2 = t1.f1;
+f1
+
+drop table t1, t2;
+#
+# MDEV-16808 Assertion on compressed blob as key field
+#
+set join_cache_level= 3;
+create table t1 (col_blob text) engine=innodb;
+create table t2 (col_blob text compressed) engine=innodb;
+select * from t1 join t2 using ( col_blob );
+col_blob
+drop tables t1, t2;
+create table t (a text compressed,b text) engine=innodb;
+create table t4 like t;
+set session join_cache_level=3;
+select * from (select * from t) as t natural join (select * from t) as t1;
+a	b
+drop tables t, t4;
 # End of 10.5 tests

mysql-test/main/column_compression.test

@@ -519,4 +519,57 @@ INSERT INTO t1 VALUES ('c','n'),('d','mmmmmmmmmm');
 SELECT GROUP_CONCAT( b, a ORDER BY 2 ) AS f FROM t1;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-24726 Assertion `0' failed in Field_varstring_compressed::key_cmp
+--echo #
+
+--echo # VARCHAR
+create table t1 (a varchar(8) compressed) character set utf8mb4;
+create algorithm=temptable view v1 as select * from t1;
+insert into t1 values ('foo'),('bar'),('foo');
+select * from v1 where a in (select a from t1);
+# cleanup
+drop view v1;
+drop table t1;
+
+create table t1 (f1 varchar(8)) charset=eucjpms collate=eucjpms_nopad_bin;
+insert into t1 values ('');
+create table t2 (f2 varchar(8) compressed) charset=eucjpms collate=eucjpms_nopad_bin;
+insert into t2 values ('a'),('b');
+select t1.* from t1 left join (select distinct f2 from t2) sq on sq.f2 = t1.f1;
+# cleanup
+drop table t1, t2;
+
+--echo # BLOB
+create table t1 (a text compressed) character set utf8mb4;
+create algorithm=temptable view v1 as select * from t1;
+insert into t1 values ('foo'),('bar'),('foo');
+select * from v1 where a in (select a from t1);
+# cleanup
+drop view v1;
+drop table t1;
+
+create table t1 (f1 text) charset=eucjpms collate=eucjpms_nopad_bin;
+insert into t1 values ('');
+create table t2 (f2 text compressed) charset=eucjpms collate=eucjpms_nopad_bin;
+insert into t2 values ('a'),('b');
+select t1.* from t1 left join (select distinct f2 from t2) sq on sq.f2 = t1.f1;
+# cleanup
+drop table t1, t2;
+
+--echo #
+--echo # MDEV-16808 Assertion on compressed blob as key field
+--echo #
+set join_cache_level= 3;
+create table t1 (col_blob text) engine=innodb;
+create table t2 (col_blob text compressed) engine=innodb;
+select * from t1 join t2 using ( col_blob );
+drop tables t1, t2;
+
+create table t (a text compressed,b text) engine=innodb;
+create table t4 like t;
+set session join_cache_level=3;
+select * from (select * from t) as t natural join (select * from t) as t1;
+drop tables t, t4;
+
 --echo # End of 10.5 tests

sql/field.cc

@@ -8374,6 +8374,59 @@ Field *Field_varstring::make_new_field(MEM_ROOT *root, TABLE *new_table,
 }
 
 
+Field *Field_varstring_compressed::make_new_field(MEM_ROOT *root, TABLE *new_table,
+                                                  bool keep_type)
+{
+  Field_varstring *res;
+  if (new_table->s->is_optimizer_tmp_table())
+  {
+    /*
+      Compressed field cannot be part of a key. For optimizer temporary
+      table we create uncompressed substitute.
+    */
+    res= new (root) Field_varstring(ptr, field_length, length_bytes, null_ptr,
+                                    null_bit, Field::NONE, &field_name,
+                                    new_table->s, charset());
+    if (res)
+    {
+      res->init_for_make_new_field(new_table, orig_table);
+      /* See Column_definition::create_length_to_internal_length_string() */
+      res->field_length--;
+    }
+  }
+  else
+    res= (Field_varstring*) Field::make_new_field(root, new_table, keep_type);
+  if (res)
+    res->length_bytes= length_bytes;
+  return res;
+}
+
+Field *Field_blob_compressed::make_new_field(MEM_ROOT *root, TABLE *new_table,
+                                                  bool keep_type)
+{
+  Field_blob *res;
+  if (new_table->s->is_optimizer_tmp_table())
+  {
+    /*
+      Compressed field cannot be part of a key. For optimizer temporary
+      table we create uncompressed substitute.
+    */
+    res= new (root) Field_blob(ptr, null_ptr, null_bit, Field::NONE, &field_name,
+                               new_table->s, packlength, charset());
+    if (res)
+    {
+      res->init_for_make_new_field(new_table, orig_table);
+      /* See Column_definition::create_length_to_internal_length_string() */
+      res->field_length--;
+    }
+  }
+  else
+    res= (Field_blob *) Field::make_new_field(root, new_table, keep_type);
+  return res;
+}
+
+
+
 Field *Field_varstring::new_key_field(MEM_ROOT *root, TABLE *new_table,
                                       uchar *new_ptr, uint32 length,
                                       uchar *new_null_ptr, uint new_null_bit)

sql/field.h

@@ -4309,6 +4309,7 @@ private:
   { DBUG_ASSERT(0); return 0; }
   using Field_varstring::key_cmp;
   Binlog_type_info binlog_type_info() const override;
+  Field *make_new_field(MEM_ROOT *root, TABLE *new_table, bool keep_type) override;
 };
 
 
@@ -4750,6 +4751,7 @@ private:
     override
   { DBUG_ASSERT(0); return 0; }
   Binlog_type_info binlog_type_info() const override;
+  Field *make_new_field(MEM_ROOT *root, TABLE *new_table, bool keep_type) override;
 };
 
 

sql/sql_select.cc

@@ -7082,7 +7082,13 @@ add_key_part(DYNAMIC_ARRAY *keyuse_array, KEY_FIELD *key_field)
         }
       }
     }
-    if (field->hash_join_is_possible() &&
+    /*
+      Compressed field cannot be part of a key. For optimizer temporary table
+      compressed fields are replaced by uncompressed, see
+      is_optimizer_tmp_table() and Field_*_compressed::make_new_field().
+    */
+    if (!field->compression_method() &&
+        field->hash_join_is_possible() &&
         (key_field->optimize & KEY_OPTIMIZE_EQ) &&
         key_field->val->used_tables())
     {

sql/table.h

@@ -1092,6 +1092,11 @@ struct TABLE_SHARE
     return (tmp_table == SYSTEM_TMP_TABLE) ? 0 : table_map_id;
   }
 
+  bool is_optimizer_tmp_table()
+  {
+    return tmp_table == INTERNAL_TMP_TABLE && !db.length && table_name.length;
+  }
+
   bool visit_subgraph(Wait_for_flush *waiting_ticket,
                       MDL_wait_for_graph_visitor *gvisitor);