MDEV-24033: SIGSEGV in __memcmp_avx2_movbe from queue_insert | SIGSEGV in __memcmp_avx2_movbe from native_compare

The issue here was the system variable max_sort_length was being applied
to decimals and it was truncating the value for decimals to the number
of bytes set by max_sort_length.
This was leading to a buffer overflow as the values were written
to the buffer without truncation and then we moved the offset to
the number of bytes(set by max_sort_length), that are needed for comparison.

The fix is to not apply max_sort_length for fixed size types like INT,
DECIMALS and only apply max_sort_length for CHAR, VARCHARS, TEXT and
BLOBS.

mysql-test/r/order_by.result

@@ -3432,4 +3432,24 @@ NULLIF(GROUP_CONCAT(v1), null)
 C
 B
 DROP TABLE t1;
+#
+# MDEV-24033: SIGSEGV in __memcmp_avx2_movbe from queue_insert | SIGSEGV in __memcmp_avx2_movbe from native_compare
+#
+SET @save_max_length_for_sort_data=@@max_length_for_sort_data;
+SET @save_max_sort_length= @@max_sort_length;
+SET @save_sql_select_limit= @@sql_select_limit;
+CREATE TABLE t1 (a DECIMAL(64,0), b INT);
+INSERT INTO t1 VALUES (1,1), (2,2), (3,3), (4,4);
+SET max_length_for_sort_data= 30;
+SET sql_select_limit = 3;
+SET max_sort_length=8;
+SELECT * FROM t1 ORDER BY a+1;
+a	b
+1	1
+2	2
+3	3
+SET max_length_for_sort_data=@save_max_length_for_sort_data;
+SET max_sort_length= @save_max_sort_length;
+SET sql_select_limit= @save_sql_select_limit;
+DROP TABLE t1;
 # End of 10.2 tests

mysql-test/t/order_by.test

@@ -2272,4 +2272,25 @@ ORDER BY id+1 DESC;
 
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-24033: SIGSEGV in __memcmp_avx2_movbe from queue_insert | SIGSEGV in __memcmp_avx2_movbe from native_compare
+--echo #
+
+SET @save_max_length_for_sort_data=@@max_length_for_sort_data;
+SET @save_max_sort_length= @@max_sort_length;
+SET @save_sql_select_limit= @@sql_select_limit;
+
+CREATE TABLE t1 (a DECIMAL(64,0), b INT);
+INSERT INTO t1 VALUES (1,1), (2,2), (3,3), (4,4);
+
+SET max_length_for_sort_data= 30;
+SET sql_select_limit = 3;
+SET max_sort_length=8;
+SELECT * FROM t1 ORDER BY a+1;
+
+SET max_length_for_sort_data=@save_max_length_for_sort_data;
+SET max_sort_length= @save_max_sort_length;
+SET sql_select_limit= @save_sql_select_limit;
+DROP TABLE t1;
+
 --echo # End of 10.2 tests

sql/field.h

@@ -1339,6 +1339,8 @@ public:
   virtual uint max_packed_col_length(uint max_length)
   { return max_length;}
 
+  virtual bool is_packable() const { return false; }
+
   uint offset(uchar *record) const
   {
     return (uint) (ptr - record);
@@ -1827,6 +1829,7 @@ public:
   bool can_optimize_range(const Item_bool_func *cond,
                           const Item *item,
                           bool is_eq_func) const;
+  bool is_packable() const { return true; }
 };
 
 /* base class for float and double and decimal (old one) */

sql/filesort.cc

@@ -1971,7 +1971,14 @@ sortlength(THD *thd, SORT_FIELD *sortorder, uint s_length,
     if (sortorder->field)
     {
       CHARSET_INFO *cs= sortorder->field->sort_charset();
+      sortorder->type= sortorder->field->is_packable() ?
+                       SORT_FIELD_ATTR::VARIABLE_SIZE :
+                       SORT_FIELD_ATTR::FIXED_SIZE;
+
       sortorder->length= sortorder->field->sort_length();
+      if (sortorder->is_variable_sized())
+        set_if_smaller(sortorder->length, thd->variables.max_sort_length);
+
       if (use_strnxfrm((cs=sortorder->field->sort_charset())))
       {
         *multi_byte_charset= true;
@@ -1982,6 +1989,10 @@ sortlength(THD *thd, SORT_FIELD *sortorder, uint s_length,
     }
     else
     {
+      sortorder->type= sortorder->item->type_handler()->is_packable() ?
+                       SORT_FIELD_ATTR::VARIABLE_SIZE :
+                       SORT_FIELD_ATTR::FIXED_SIZE;
+
       sortorder->item->sortlength(thd, sortorder->item, sortorder);
       if (use_strnxfrm(sortorder->item->collation.collation))
       {
@@ -1990,7 +2001,8 @@ sortlength(THD *thd, SORT_FIELD *sortorder, uint s_length,
       if (sortorder->item->maybe_null)
 	length++;				// Place for NULL marker
     }
-    set_if_smaller(sortorder->length, thd->variables.max_sort_length);
+    if (sortorder->is_variable_sized())
+      set_if_smaller(sortorder->length, thd->variables.max_sort_length);
     length+=sortorder->length;
   }
   sortorder->field= (Field*) 0;			// end marker

sql/sql_class.h

@@ -5449,6 +5449,8 @@ struct SORT_FIELD_ATTR
 {
   uint length;          /* Length of sort field */
   uint suffix_length;   /* Length suffix (0-4) */
+  enum Type { FIXED_SIZE, VARIABLE_SIZE } type;
+  bool is_variable_sized() { return type == VARIABLE_SIZE; }
 };
 
 

sql/sql_type.h

@@ -92,6 +92,7 @@ public:
   virtual void sortlength(THD *thd,
                           const Type_std_attributes *item,
                           SORT_FIELD_ATTR *attr) const= 0;
+  virtual bool is_packable() const { return false; }
 };
 
 
@@ -169,6 +170,7 @@ public:
   void sortlength(THD *thd,
                   const Type_std_attributes *item,
                   SORT_FIELD_ATTR *attr) const;
+  bool is_packable()const { return true; }
 };