MDEV-11598 Assertion `!table || (!table->read_set || bitmap_is_set(table->read_set, field_index))' failed

Found and fixed 2 problems:

- Filesort addon fields didn't mark virtual columns properly
- multi-range-read calculated vcol bitmap but was not using it.
  This caused wrong vcol field to be calculated on read, which caused the assert.

mysql-test/suite/vcol/inc/vcol_keys.inc

@@ -181,3 +181,54 @@ create table t1 (a int, b double as (rand()));
 --error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED
 alter table t1 add index (b);
 drop table t1;
+
+#
+# MDEV-11598 Assertion `!table || (!table->read_set... failed
+#
+
+CREATE OR REPLACE TABLE t1 (
+    f2 DOUBLE NOT NULL DEFAULT '0',
+    f3 DOUBLE NOT NULL DEFAULT '0',
+    f4 DOUBLE,
+    f5 DOUBLE DEFAULT '0',
+    v4 DOUBLE AS (IF(f4,f3,f2)) VIRTUAL,
+    KEY (f5),
+    KEY (v4)
+);
+
+INSERT INTO t1 (f2,f3,f4,f5) VALUES (5,4,1,0),(5,7,NULL,0);
+INSERT INTO t1 (f2,f3,f4,f5) SELECT f2, f3, f5, f3 FROM t1;
+INSERT INTO t1 (f2,f3,f4,f5) VALUES (5,0,NULL,1);
+INSERT INTO t1 (f2,f3,f4,f5) SELECT f2, f5, f5, f3 FROM t1;
+DELETE FROM t1 WHERE f5 = 1 OR v4 = 4 ORDER BY f5,v4 LIMIT 9;
+SELECT * from t1;
+DROP TABLE t1;
+
+# Another similar failure
+
+CREATE TABLE t1 (
+ d DECIMAL(63,0) NOT NULL DEFAULT 0,
+ c VARCHAR(64) NOT NULL DEFAULT '',
+ vd DECIMAL(63,0) AS (d) VIRTUAL,
+ vc VARCHAR(2048) AS (c) VIRTUAL,
+ pk BIGINT AUTO_INCREMENT,
+ PRIMARY KEY(pk));
+
+INSERT INTO t1 (d,c) VALUES (0.5,'foo');
+SELECT * FROM t1 WHERE vc != 'bar' ORDER BY vd;
+DROP TABLE t1;
+
+#
+# MDEV-11729:  Crash when using partial indexed virtual fields
+#
+
+CREATE TABLE t1 (
+ pk BIGINT,
+ c CHAR(64) NOT NULL DEFAULT '',
+ vc CHAR(64) AS (c) VIRTUAL,
+ PRIMARY KEY(pk),
+ INDEX(vc(32))
+);
+DELETE FROM t1 WHERE vc IS NULL ORDER BY pk;
+DROP TABLE t1;
+

mysql-test/suite/vcol/r/vcol_keys_innodb.result

@@ -168,3 +168,50 @@ create table t1 (a int, b double as (rand()));
 alter table t1 add index (b);
 ERROR HY000: Function or expression 'rand()' cannot be used in the GENERATED ALWAYS AS clause of `b`
 drop table t1;
+CREATE OR REPLACE TABLE t1 (
+f2 DOUBLE NOT NULL DEFAULT '0',
+f3 DOUBLE NOT NULL DEFAULT '0',
+f4 DOUBLE,
+f5 DOUBLE DEFAULT '0',
+v4 DOUBLE AS (IF(f4,f3,f2)) VIRTUAL,
+KEY (f5),
+KEY (v4)
+);
+INSERT INTO t1 (f2,f3,f4,f5) VALUES (5,4,1,0),(5,7,NULL,0);
+INSERT INTO t1 (f2,f3,f4,f5) SELECT f2, f3, f5, f3 FROM t1;
+INSERT INTO t1 (f2,f3,f4,f5) VALUES (5,0,NULL,1);
+INSERT INTO t1 (f2,f3,f4,f5) SELECT f2, f5, f5, f3 FROM t1;
+DELETE FROM t1 WHERE f5 = 1 OR v4 = 4 ORDER BY f5,v4 LIMIT 9;
+SELECT * from t1;
+f2	f3	f4	f5	v4
+5	7	NULL	0	5
+5	4	0	4	5
+5	7	0	7	5
+5	0	0	4	5
+5	0	0	7	5
+5	7	7	7	7
+5	1	1	0	1
+DROP TABLE t1;
+CREATE TABLE t1 (
+d DECIMAL(63,0) NOT NULL DEFAULT 0,
+c VARCHAR(64) NOT NULL DEFAULT '',
+vd DECIMAL(63,0) AS (d) VIRTUAL,
+vc VARCHAR(2048) AS (c) VIRTUAL,
+pk BIGINT AUTO_INCREMENT,
+PRIMARY KEY(pk));
+INSERT INTO t1 (d,c) VALUES (0.5,'foo');
+Warnings:
+Note	1265	Data truncated for column 'd' at row 1
+SELECT * FROM t1 WHERE vc != 'bar' ORDER BY vd;
+d	c	vd	vc	pk
+1	foo	1	foo	1
+DROP TABLE t1;
+CREATE TABLE t1 (
+pk BIGINT,
+c CHAR(64) NOT NULL DEFAULT '',
+vc CHAR(64) AS (c) VIRTUAL,
+PRIMARY KEY(pk),
+INDEX(vc(32))
+);
+DELETE FROM t1 WHERE vc IS NULL ORDER BY pk;
+DROP TABLE t1;

mysql-test/suite/vcol/r/vcol_keys_myisam.result

@@ -278,3 +278,50 @@ create table t1 (a int, b double as (rand()));
 alter table t1 add index (b);
 ERROR HY000: Function or expression 'rand()' cannot be used in the GENERATED ALWAYS AS clause of `b`
 drop table t1;
+CREATE OR REPLACE TABLE t1 (
+f2 DOUBLE NOT NULL DEFAULT '0',
+f3 DOUBLE NOT NULL DEFAULT '0',
+f4 DOUBLE,
+f5 DOUBLE DEFAULT '0',
+v4 DOUBLE AS (IF(f4,f3,f2)) VIRTUAL,
+KEY (f5),
+KEY (v4)
+);
+INSERT INTO t1 (f2,f3,f4,f5) VALUES (5,4,1,0),(5,7,NULL,0);
+INSERT INTO t1 (f2,f3,f4,f5) SELECT f2, f3, f5, f3 FROM t1;
+INSERT INTO t1 (f2,f3,f4,f5) VALUES (5,0,NULL,1);
+INSERT INTO t1 (f2,f3,f4,f5) SELECT f2, f5, f5, f3 FROM t1;
+DELETE FROM t1 WHERE f5 = 1 OR v4 = 4 ORDER BY f5,v4 LIMIT 9;
+SELECT * from t1;
+f2	f3	f4	f5	v4
+5	7	NULL	0	5
+5	4	0	4	5
+5	7	0	7	5
+5	0	0	4	5
+5	0	0	7	5
+5	7	7	7	7
+5	1	1	0	1
+DROP TABLE t1;
+CREATE TABLE t1 (
+d DECIMAL(63,0) NOT NULL DEFAULT 0,
+c VARCHAR(64) NOT NULL DEFAULT '',
+vd DECIMAL(63,0) AS (d) VIRTUAL,
+vc VARCHAR(2048) AS (c) VIRTUAL,
+pk BIGINT AUTO_INCREMENT,
+PRIMARY KEY(pk));
+INSERT INTO t1 (d,c) VALUES (0.5,'foo');
+Warnings:
+Note	1265	Data truncated for column 'd' at row 1
+SELECT * FROM t1 WHERE vc != 'bar' ORDER BY vd;
+d	c	vd	vc	pk
+1	foo	1	foo	1
+DROP TABLE t1;
+CREATE TABLE t1 (
+pk BIGINT,
+c CHAR(64) NOT NULL DEFAULT '',
+vc CHAR(64) AS (c) VIRTUAL,
+PRIMARY KEY(pk),
+INDEX(vc(32))
+);
+DELETE FROM t1 WHERE vc IS NULL ORDER BY pk;
+DROP TABLE t1;

sql/field.cc

@@ -10864,3 +10864,14 @@ bool Field::save_in_field_ignore_value(bool view_error_processing)
     return save_in_field_default_value(view_error_processing);
   return 0; // ignore
 }
+
+
+void Field::register_field_in_read_map()
+{
+  if (vcol_info)
+  {
+    Item *vcol_item= vcol_info->expr;
+    vcol_item->walk(&Item::register_field_in_read_map, 1, 0);
+  }
+  bitmap_set_bit(table->read_set, field_index);
+}

sql/field.h

@@ -1475,6 +1475,9 @@ public:
   bool save_in_field_default_value(bool view_eror_processing);
   bool save_in_field_ignore_value(bool view_error_processing);
 
+  /* Mark field in read map. Updates also virtual fields */
+  void register_field_in_read_map();
+
   friend int cre_myisam(char * name, register TABLE *form, uint options,
 			ulonglong auto_increment_value);
   friend class Copy_field;

sql/filesort.cc

@@ -716,7 +716,7 @@ static ha_rows find_all_keys(THD *thd, Sort_param *param, SQL_SELECT *select,
   TABLE *sort_form;
   handler *file;
   MY_BITMAP *save_read_set, *save_write_set, *save_vcol_set;
-  
+  Item *sort_cond;
   DBUG_ENTER("find_all_keys");
   DBUG_PRINT("info",("using: %s",
                      (select ? select->quick ? "ranges" : "where":
@@ -754,22 +754,22 @@ static ha_rows find_all_keys(THD *thd, Sort_param *param, SQL_SELECT *select,
   /* Remember original bitmaps */
   save_read_set=  sort_form->read_set;
   save_write_set= sort_form->write_set;
-  save_vcol_set= sort_form->vcol_set;
+  save_vcol_set=  sort_form->vcol_set;
+
   /* Set up temporary column read map for columns used by sort */
   bitmap_clear_all(&sort_form->tmp_set);
-  /* Temporary set for register_used_fields and register_field_in_read_map */
-  sort_form->read_set= &sort_form->tmp_set;
-  register_used_fields(param);
-  if (quick_select)
-    select->quick->add_used_key_part_to_set(sort_form->read_set);
-
-  Item *sort_cond= !select ?  
-                     0 : !select->pre_idx_push_select_cond ? 
-                           select->cond : select->pre_idx_push_select_cond;
-  if (sort_cond)
-    sort_cond->walk(&Item::register_field_in_read_map, 1, sort_form);
   sort_form->column_bitmaps_set(&sort_form->tmp_set, &sort_form->tmp_set, 
                                 &sort_form->tmp_set);
+  register_used_fields(param);
+  if (quick_select)
+    select->quick->add_used_key_part_to_set();
+
+  sort_cond= (!select ? 0 :
+              (!select->pre_idx_push_select_cond ?
+               select->cond : select->pre_idx_push_select_cond));
+  if (sort_cond)
+    sort_cond->walk(&Item::register_field_in_read_map, 1, sort_form);
+  sort_form->file->column_bitmaps_signal();
 
   if (quick_select)
   {
@@ -1259,7 +1259,6 @@ static void register_used_fields(Sort_param *param)
 {
   reg1 SORT_FIELD *sort_field;
   TABLE *table=param->sort_form;
-  MY_BITMAP *bitmap= table->read_set;
 
   for (sort_field= param->local_sortorder ;
        sort_field != param->end ;
@@ -1269,14 +1268,7 @@ static void register_used_fields(Sort_param *param)
     if ((field= sort_field->field))
     {
       if (field->table == table)
-      {
+        field->register_field_in_read_map();
-        if (field->vcol_info)
-	{
-          Item *vcol_item= field->vcol_info->expr;
-          vcol_item->walk(&Item::register_field_in_read_map, 1, 0);
-        }                   
-        bitmap_set_bit(bitmap, field->field_index);
-      }
     }
     else
     {						// Item
@@ -1289,7 +1281,7 @@ static void register_used_fields(Sort_param *param)
     SORT_ADDON_FIELD *addonf= param->addon_field;
     Field *field;
     for ( ; (field= addonf->field) ; addonf++)
-      bitmap_set_bit(bitmap, field->field_index);
+      field->register_field_in_read_map();
   }
   else
   {

sql/opt_range.cc

@@ -11101,6 +11101,7 @@ int QUICK_RANGE_SELECT::reset()
   HANDLER_BUFFER empty_buf;
   MY_BITMAP * const save_read_set= head->read_set;
   MY_BITMAP * const save_write_set= head->write_set;
+  MY_BITMAP * const save_vcol_set= head->vcol_set;
   DBUG_ENTER("QUICK_RANGE_SELECT::reset");
   last_range= NULL;
   cur_range= (QUICK_RANGE**) ranges.buffer;
@@ -11114,7 +11115,8 @@ int QUICK_RANGE_SELECT::reset()
   }
 
   if (in_ror_merged_scan)
-    head->column_bitmaps_set_no_signal(&column_bitmap, &column_bitmap);
+    head->column_bitmaps_set_no_signal(&column_bitmap, &column_bitmap,
+                                       &column_bitmap);
 
   if (file->inited == handler::NONE)
   {
@@ -11157,8 +11159,8 @@ int QUICK_RANGE_SELECT::reset()
 err:
   /* Restore bitmaps set on entry */
   if (in_ror_merged_scan)
-    head->column_bitmaps_set_no_signal(save_read_set, save_write_set);
+    head->column_bitmaps_set_no_signal(save_read_set, save_write_set,
-
+                                       save_vcol_set);
   DBUG_RETURN(error);
 }
 
@@ -11189,13 +11191,16 @@ int QUICK_RANGE_SELECT::get_next()
 
   MY_BITMAP * const save_read_set= head->read_set;
   MY_BITMAP * const save_write_set= head->write_set;
+  MY_BITMAP * const save_vcol_set= head->vcol_set;
   /*
     We don't need to signal the bitmap change as the bitmap is always the
     same for this head->file
   */
-  head->column_bitmaps_set_no_signal(&column_bitmap, &column_bitmap);
+  head->column_bitmaps_set_no_signal(&column_bitmap, &column_bitmap,
+                                     &column_bitmap);
   result= file->multi_range_read_next(&dummy);
-  head->column_bitmaps_set_no_signal(save_read_set, save_write_set);
+  head->column_bitmaps_set_no_signal(save_read_set, save_write_set,
+                                     save_vcol_set);
   DBUG_RETURN(result);
 }
 
@@ -11372,7 +11377,7 @@ QUICK_SELECT_DESC::QUICK_SELECT_DESC(QUICK_RANGE_SELECT *q,
   used_key_parts (used_key_parts_arg)
 {
   QUICK_RANGE *r;
-  /* 
+  /*
     Use default MRR implementation for reverse scans. No table engine
     currently can do an MRR scan with output in reverse index order.
   */
@@ -11847,62 +11852,76 @@ void QUICK_ROR_UNION_SELECT::add_keys_and_lengths(String *key_names,
 }
 
 
-void QUICK_RANGE_SELECT::add_used_key_part_to_set(MY_BITMAP *col_set)
+void QUICK_RANGE_SELECT::add_used_key_part_to_set()
 {
   uint key_len;
   KEY_PART *part= key_parts;
   for (key_len=0; key_len < max_used_key_length;
        key_len += (part++)->store_length)
   {
-    bitmap_set_bit(col_set, part->field->field_index);
+    /*
+      We have to use field_index instead of part->field
+      as for partial fields, part->field points to
+      a temporary field that is only part of the original
+      field.  field_index always points to the original field
+    */
+    Field *field= head->field[part->field->field_index];
+    field->register_field_in_read_map();
   }
 }
 
 
-void QUICK_GROUP_MIN_MAX_SELECT::add_used_key_part_to_set(MY_BITMAP *col_set)
+void QUICK_GROUP_MIN_MAX_SELECT::add_used_key_part_to_set()
 {
   uint key_len;
   KEY_PART_INFO *part= index_info->key_part;
   for (key_len=0; key_len < max_used_key_length;
        key_len += (part++)->store_length)
   {
-    bitmap_set_bit(col_set, part->field->field_index);
+    /*
+      We have to use field_index instead of part->field
+      as for partial fields, part->field points to
+      a temporary field that is only part of the original
+      field.  field_index always points to the original field
+    */
+    Field *field= head->field[part->field->field_index];
+    field->register_field_in_read_map();
   }
 }
 
 
-void QUICK_ROR_INTERSECT_SELECT::add_used_key_part_to_set(MY_BITMAP *col_set)
+void QUICK_ROR_INTERSECT_SELECT::add_used_key_part_to_set()
 {
   List_iterator_fast<QUICK_SELECT_WITH_RECORD> it(quick_selects);
   QUICK_SELECT_WITH_RECORD *quick;
   while ((quick= it++))
   {
-    quick->quick->add_used_key_part_to_set(col_set);
+    quick->quick->add_used_key_part_to_set();
   }
 }
 
 
-void QUICK_INDEX_SORT_SELECT::add_used_key_part_to_set(MY_BITMAP *col_set)
+void QUICK_INDEX_SORT_SELECT::add_used_key_part_to_set()
 {
   QUICK_RANGE_SELECT *quick;
   List_iterator_fast<QUICK_RANGE_SELECT> it(quick_selects);
   while ((quick= it++))
   {
-    quick->add_used_key_part_to_set(col_set);
+    quick->add_used_key_part_to_set();
   }
   if (pk_quick_select)
-    pk_quick_select->add_used_key_part_to_set(col_set);
+    pk_quick_select->add_used_key_part_to_set();
 }
 
 
-void QUICK_ROR_UNION_SELECT::add_used_key_part_to_set(MY_BITMAP *col_set)
+void QUICK_ROR_UNION_SELECT::add_used_key_part_to_set()
 {
   QUICK_SELECT_I *quick;
   List_iterator_fast<QUICK_SELECT_I> it(quick_selects);
 
   while ((quick= it++))
   {
-    quick->add_used_key_part_to_set(col_set);
+    quick->add_used_key_part_to_set();
   }
 }
 

sql/opt_range.h

@@ -1006,7 +1006,7 @@ public:
 
     This is used by an optimization in filesort.
   */
-  virtual void add_used_key_part_to_set(MY_BITMAP *col_set)=0;
+  virtual void add_used_key_part_to_set()=0;
 };
 
 
@@ -1097,7 +1097,7 @@ public:
   virtual void replace_handler(handler *new_file) { file= new_file; }
   QUICK_SELECT_I *make_reverse(uint used_key_parts_arg);
 
-  virtual void add_used_key_part_to_set(MY_BITMAP *col_set);
+  virtual void add_used_key_part_to_set();
 
 private:
   /* Default copy ctor used by QUICK_SELECT_DESC */
@@ -1261,7 +1261,7 @@ public:
   /* used to get rows collected in Unique */
   READ_RECORD read_record;
 
-  virtual void add_used_key_part_to_set(MY_BITMAP *col_set);
+  virtual void add_used_key_part_to_set();
 };
 
 
@@ -1336,7 +1336,7 @@ public:
   void add_keys_and_lengths(String *key_names, String *used_lengths);
   Explain_quick_select *get_explain(MEM_ROOT *alloc);
   bool is_keys_used(const MY_BITMAP *fields);
-  void add_used_key_part_to_set(MY_BITMAP *col_set);
+  void add_used_key_part_to_set();
 #ifndef DBUG_OFF
   void dbug_dump(int indent, bool verbose);
 #endif
@@ -1416,7 +1416,7 @@ public:
   void add_keys_and_lengths(String *key_names, String *used_lengths);
   Explain_quick_select *get_explain(MEM_ROOT *alloc);
   bool is_keys_used(const MY_BITMAP *fields);
-  void add_used_key_part_to_set(MY_BITMAP *col_set);
+  void add_used_key_part_to_set();
 #ifndef DBUG_OFF
   void dbug_dump(int indent, bool verbose);
 #endif
@@ -1560,7 +1560,7 @@ public:
   bool unique_key_range() { return false; }
   int get_type() { return QS_TYPE_GROUP_MIN_MAX; }
   void add_keys_and_lengths(String *key_names, String *used_lengths);
-  void add_used_key_part_to_set(MY_BITMAP *col_set);
+  void add_used_key_part_to_set();
 #ifndef DBUG_OFF
   void dbug_dump(int indent, bool verbose);
 #endif