MDEV-22728: SIGFPE in Unique::get_cost_calc_buff_size from prepare_search_best_index_intersect on optimized builds

For low sort_buffer_size, in the cost calculation of using the Unique object the elements in the tree were evaluated to 0, make sure to have atleast 1 element in the Unique tree.

Also for the function Unique::get allocate memory for atleast MERGEBUFF2+1 keys.

mysql-test/r/index_merge_innodb.result

@@ -805,3 +805,33 @@ sum(col1)
 33632261
 drop table t1;
 set optimizer_switch=@tmp_optimizer_switch;
+#
+# MDEV-22728: SIGFPE in Unique::get_cost_calc_buff_size from prepare_search_best_index_intersect
+# on optimized builds
+#
+SET @save_sort_buffer_size=@@sort_buffer_size;
+SET @save_innodb_file_format= @@innodb_file_format;
+SET @save_innodb_large_prefix= @@innodb_large_prefix;
+SET sort_buffer_size=2048;
+SET GLOBAL innodb_file_format = BARRACUDA;
+SET GLOBAL innodb_large_prefix = ON;
+CREATE TABLE t1 (
+a VARCHAR(1024) CHARACTER SET UTF8 PRIMARY KEY,
+b INT,
+c INT,
+INDEX (b)
+) ENGINE=InnoDB CHARSET utf8 ROW_FORMAT= DYNAMIC;
+INSERT INTO t1 SELECT seq, seq, seq from seq_1_to_100;
+EXPLAIN SELECT * FROM t1 WHERE a='1' OR b < 5;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index_merge	PRIMARY,b	b,PRIMARY	5,3074	NULL	4	Using sort_union(b,PRIMARY); Using where
+SELECT * FROM t1 WHERE a='1' OR b < 5;
+a	b	c
+2	2	2
+3	3	3
+4	4	4
+1	1	1
+DROP TABLE t1;
+SET GLOBAL innodb_file_format = @save_innodb_file_format;
+SET GLOBAL innodb_large_prefix = @save_innodb_large_prefix;
+SET sort_buffer_size= @save_sort_buffer_size;

mysql-test/t/index_merge_innodb.test

@@ -198,4 +198,31 @@ SELECT sum(col1) FROM t1 FORCE INDEX (key1,key2) WHERE  (key1 between 10 and 819
 drop table t1;
 set optimizer_switch=@tmp_optimizer_switch;
 
+--echo #
+--echo # MDEV-22728: SIGFPE in Unique::get_cost_calc_buff_size from prepare_search_best_index_intersect
+--echo # on optimized builds
+--echo #
+
+SET @save_sort_buffer_size=@@sort_buffer_size;
+SET @save_innodb_file_format= @@innodb_file_format;
+SET @save_innodb_large_prefix= @@innodb_large_prefix;
+SET sort_buffer_size=2048;
+SET GLOBAL innodb_file_format = BARRACUDA;
+SET GLOBAL innodb_large_prefix = ON;
+
+CREATE TABLE t1 (
+  a VARCHAR(1024) CHARACTER SET UTF8 PRIMARY KEY,
+  b INT,
+  c INT,
+  INDEX (b)
+) ENGINE=InnoDB CHARSET utf8 ROW_FORMAT= DYNAMIC;
+INSERT INTO t1 SELECT seq, seq, seq from seq_1_to_100;
+EXPLAIN SELECT * FROM t1 WHERE a='1' OR b < 5;
+SELECT * FROM t1 WHERE a='1' OR b < 5;
+DROP TABLE t1;
+
+SET GLOBAL innodb_file_format = @save_innodb_file_format;
+SET GLOBAL innodb_large_prefix = @save_innodb_large_prefix;
+SET sort_buffer_size= @save_sort_buffer_size;
+
 disconnect disable_purge;

sql/sql_class.h

@@ -5216,6 +5216,9 @@ public:
   {
     ulonglong max_elems_in_tree=
       max_in_memory_size / ALIGN_SIZE(sizeof(TREE_ELEMENT)+key_size);
+
+    if (max_elems_in_tree == 0)
+      max_elems_in_tree= 1;
     return (int) (sizeof(uint)*(1 + nkeys/max_elems_in_tree));
   }
 

sql/uniques.cc

@@ -317,6 +317,9 @@ double Unique::get_use_cost(uint *buffer, size_t nkeys, uint key_size,
   max_elements_in_tree= ((size_t) max_in_memory_size /
                          ALIGN_SIZE(sizeof(TREE_ELEMENT)+key_size));
 
+  if (max_elements_in_tree == 0)
+    max_elements_in_tree= 1;
+
   n_full_trees=    nkeys / max_elements_in_tree;
   last_tree_elems= nkeys % max_elements_in_tree;
 
@@ -781,7 +784,13 @@ bool Unique::get(TABLE *table)
   /* Not enough memory; Save the result to file && free memory used by tree */
   if (flush())
     return 1;
-  size_t buff_sz= (max_in_memory_size / full_size + 1) * full_size;
+
+  /*
+    merge_buffer must fit at least MERGEBUFF2 + 1 keys, because
+    merge_index() can merge that many BUFFPEKs at once. The extra space for
+    one key for Sort_param::unique_buff
+  */
+  size_t buff_sz= MY_MAX(MERGEBUFF2+1, max_in_memory_size/full_size+1) * full_size;
   if (!(sort_buffer= (uchar*) my_malloc(buff_sz, MYF(MY_THREAD_SPECIFIC|MY_WME))))
     return 1;