MDEV-35469 Heap tables are calling mallocs to often

Heap tables are allocated blocks to store rows according to
my_default_record_cache (mapped to the server global variable
 read_buffer_size).
This causes performance issues when the record length is big
(> 1000 bytes) and the my_default_record_cache is small.

Changed to instead split the default heap allocation to 1/16 of the
allowed space and not use my_default_record_cache anymore when creating
the heap. The allocation is also aligned to be just under a power of 2.

For some test that I have been running, which was using record length=633,
the speed of the query doubled thanks to this change.

Other things:
- Fixed calculation of max_records passed to hp_create() to take
  into account padding between records.
- Updated calculation of memory needed by heap tables. Before we
  did not take into account internal structures needed to access rows.
- Changed block sized for memory_table from 1 to 16384 to get less
  fragmentation. This also avoids a problem where we need 1K
  to manage index and row storage which was not counted for before.
- Moved heap memory usage to a separate test for 32 bit.
- Allocate all data blocks in heap in powers of 2. Change reported
  memory usage for heap to reflect this.

Reviewed-by: Sergei Golubchik <serg@mariadb.org>

include/heap.h

@@ -105,6 +105,7 @@ typedef struct st_heap_block
   uint recbuffer;			/* Length of one saved record */
   ulong records_in_block;		/* Records in one heap-block */
   ulong last_allocated; /* number of records there is allocated space for */
+  size_t alloc_size;			/* Allocate blocks of this size */
 } HP_BLOCK;
 
 struct st_heap_info;			/* For reference */

include/my_global.h

@@ -670,7 +670,7 @@ typedef SOCKET_SIZE_TYPE size_socket;
   How much overhead does malloc have. The code often allocates
   something like 1024-MALLOC_OVERHEAD bytes
 */
-#define MALLOC_OVERHEAD 8
+#define MALLOC_OVERHEAD (8+24)
 
 	/* get memory in huncs */
 #define ONCE_ALLOC_INIT		(uint) 4096

mysql-test/main/count_distinct.result

@@ -97,6 +97,8 @@ drop view v1;
 create table t1 (user_id char(64) character set utf8);
 insert t1 values(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17);
 set @@tmp_table_size = 1024;
+Warnings:
+Warning	1292	Truncated incorrect tmp_table_size value: '1024'
 select count(distinct user_id) from t1;
 count(distinct user_id)
 17
@@ -126,6 +128,8 @@ insert into t1 values
 ( 2 , 13 ),
 ( 3 , 14 );
 set @@tmp_table_size=1024;
+Warnings:
+Warning	1292	Truncated incorrect tmp_table_size value: '1024'
 select count(distinct a) from t1;
 count(distinct a)
 10

mysql-test/main/derived_opt.result

@@ -310,7 +310,7 @@ a char(2) NOT NULL DEFAULT '',
 PRIMARY KEY (a)
 ) ENGINE=MyISAM;
 INSERT INTO t4 VALUES ('CD');
-set @@tmp_table_size=8192;
+set @@tmp_table_size=16384;
 EXPLAIN
 SELECT * FROM t3 AS tx JOIN t2 AS ty ON (tx.pk = ty.pk)
 WHERE

mysql-test/main/derived_opt.test

@@ -245,7 +245,7 @@ CREATE TABLE t4 (
 ) ENGINE=MyISAM;
 INSERT INTO t4 VALUES ('CD');
 
-set @@tmp_table_size=8192;
+set @@tmp_table_size=16384;
 
 --replace_column 9 #
 EXPLAIN

mysql-test/main/derived_view.result

@@ -2183,6 +2183,8 @@ INSERT INTO t3 VALUES ('Miami');
 SET @save_optimizer_switch=@@optimizer_switch;
 SET optimizer_switch = 'derived_with_keys=on';
 SET @@tmp_table_size=1024*4;
+Warnings:
+Warning	1292	Truncated incorrect tmp_table_size value: '4096'
 explain SELECT * FROM (SELECT t1.* FROM t1, t2) AS t JOIN t3 ON t3.a = t.b;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t3	system	NULL	NULL	NULL	NULL	1	

mysql-test/main/distinct.result

@@ -1005,6 +1005,8 @@ DROP TABLE t1;
 #
 SET @tmp_table_size_save= @@tmp_table_size;
 SET @@tmp_table_size= 1024;
+Warnings:
+Warning	1292	Truncated incorrect tmp_table_size value: '1024'
 CREATE TABLE t1 (a INT);
 INSERT INTO t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8);
 INSERT INTO t1 SELECT a+8 FROM t1;

mysql-test/main/error_simulation.result

@@ -1,15 +1,12 @@
 CREATE TABLE t1 (
-a varchar(32) character set utf8 collate utf8_bin NOT NULL,
+a varchar(128) character set utf8 collate utf8_bin NOT NULL,
-b varchar(32) character set utf8 collate utf8_bin NOT NULL )
+b varchar(128) character set utf8 collate utf8_bin NOT NULL )
 ENGINE=MyISAM DEFAULT CHARSET=utf8;
-INSERT INTO t1 VALUES
+INSERT INTO t1
-('AAAAAAAAAA','AAAAAAAAAA'), ('AAAAAAAAAB','AAAAAAAAAB '),
+select concat(repeat("A", 50),char(32+mod(seq,31)),char(32+mod(seq,29))),
-('AAAAAAAAAB','AAAAAAAAAB'), ('AAAAAAAAAC','AAAAAAAAAC'),
+concat(repeat("A", 50),char(32+mod(seq,31)),char(32+mod(seq,29)))
-('AAAAAAAAAD','AAAAAAAAAD'), ('AAAAAAAAAE','AAAAAAAAAE'),
+from seq_1_to_128;
-('AAAAAAAAAF','AAAAAAAAAF'), ('AAAAAAAAAG','AAAAAAAAAG'),
+set tmp_table_size=16384;
-('AAAAAAAAAH','AAAAAAAAAH'), ('AAAAAAAAAI','AAAAAAAAAI'),
-('AAAAAAAAAJ','AAAAAAAAAJ'), ('AAAAAAAAAK','AAAAAAAAAK');
-set tmp_table_size=1024;
 SET @saved_dbug = @@SESSION.debug_dbug;
 set session debug_dbug="+d,raise_error";
 SELECT MAX(a) FROM t1 GROUP BY a,b;

mysql-test/main/error_simulation.test

@@ -1,23 +1,20 @@
 --source include/have_debug.inc
 --source include/not_embedded.inc
-
+--source include/have_sequence.inc
 #
 # Bug #28499: crash for grouping query when tmp_table_size is too small
 #
 CREATE TABLE t1 (
-  a varchar(32) character set utf8 collate utf8_bin NOT NULL,
+  a varchar(128) character set utf8 collate utf8_bin NOT NULL,
-  b varchar(32) character set utf8 collate utf8_bin NOT NULL )
+  b varchar(128) character set utf8 collate utf8_bin NOT NULL )
 ENGINE=MyISAM DEFAULT CHARSET=utf8;
 
-INSERT INTO t1 VALUES
+INSERT INTO t1
-  ('AAAAAAAAAA','AAAAAAAAAA'), ('AAAAAAAAAB','AAAAAAAAAB '),
+select concat(repeat("A", 50),char(32+mod(seq,31)),char(32+mod(seq,29))),
-  ('AAAAAAAAAB','AAAAAAAAAB'), ('AAAAAAAAAC','AAAAAAAAAC'),
+       concat(repeat("A", 50),char(32+mod(seq,31)),char(32+mod(seq,29)))
-  ('AAAAAAAAAD','AAAAAAAAAD'), ('AAAAAAAAAE','AAAAAAAAAE'),
+       from seq_1_to_128;
-  ('AAAAAAAAAF','AAAAAAAAAF'), ('AAAAAAAAAG','AAAAAAAAAG'),
-  ('AAAAAAAAAH','AAAAAAAAAH'), ('AAAAAAAAAI','AAAAAAAAAI'),
-  ('AAAAAAAAAJ','AAAAAAAAAJ'), ('AAAAAAAAAK','AAAAAAAAAK');
 
-set tmp_table_size=1024;
+set tmp_table_size=16384;
 
 # Set debug flag so an error is returned when
 # tmp table in query is converted from heap to myisam

mysql-test/main/order_by.result

@@ -4331,6 +4331,8 @@ CREATE TABLE t1(a VARCHAR(1027), b INT);
 INSERT INTO t1 SELECT seq, seq from seq_1_to_34;
 SET @save_tmp_memory_table_size= @@tmp_memory_table_size;
 SET tmp_memory_table_size= 1056*2;
+Warnings:
+Warning	1292	Truncated incorrect tmp_memory_table_size value: '2112'
 SELECT COUNT(DISTINCT a) FROM t1;
 COUNT(DISTINCT a)
 34

mysql-test/main/set_operation.result

@@ -1085,10 +1085,8 @@ select 1 union all select 2 union all select 3 union select 4;
 3
 4
 # test with limited resource
-set @@max_heap_table_size= 1024;
+set @@max_heap_table_size= 16384;
-Warnings:
+set @@tmp_table_size= 16384;
-Warning	1292	Truncated incorrect max_heap_table_size value: '1024'
-set @@tmp_table_size= 1024;
 create table t1 (a int, b int);
 insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(0,0);
 insert into t1 select * from t1;

mysql-test/main/set_operation.test

@@ -457,8 +457,8 @@ select 1 union all select 2 union all select 3 union select 4;
 
 --echo # test with limited resource
 
-set @@max_heap_table_size= 1024;
+set @@max_heap_table_size= 16384;
-set @@tmp_table_size= 1024;
+set @@tmp_table_size= 16384;
 
 create table t1 (a int, b int);
 insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(0,0);

mysql-test/main/union.result

@@ -874,7 +874,7 @@ insert into t1 select * from t2;
 insert into t2 select * from t1;
 insert into t1 select * from t2;
 insert into t2 select * from t1;
-set local tmp_table_size=1024;
+set local tmp_table_size=16384;
 select count(*) from (select * from t1 union all select * from t2 order by 1) b;
 count(*)
 21

mysql-test/main/union.test

@@ -503,7 +503,7 @@ insert into t1 select * from t2;
 insert into t2 select * from t1;
 insert into t1 select * from t2;
 insert into t2 select * from t1;
-set local tmp_table_size=1024;
+set local tmp_table_size=16384;
 select count(*) from (select * from t1 union all select * from t2 order by 1) b;
 select count(*) from t1;
 select count(*) from t2;

mysql-test/main/update.result

@@ -491,7 +491,9 @@ a	quux
 DROP TABLE t1;
 connect  con1,localhost,root,,test;
 connection con1;
-set tmp_table_size=1024;
+set tmp_table_size=2048;
+Warnings:
+Warning	1292	Truncated incorrect tmp_table_size value: '2048'
 create table t1 (id int, a int, key idx(a));
 create table t2 (id int unsigned not null auto_increment primary key, a int);
 insert into t2(a) values(1),(2),(3),(4),(5),(6),(7),(8);

mysql-test/main/update.test

@@ -397,7 +397,7 @@ DROP TABLE t1;
 connect (con1,localhost,root,,test);
 connection con1;
 
-set tmp_table_size=1024;
+set tmp_table_size=2048;
 
 # Create the test tables
 create table t1 (id int, a int, key idx(a));

mysql-test/main/variables.result

@@ -547,7 +547,7 @@ set global table_open_cache=100;
 set default_storage_engine=myisam;
 set global thread_cache_size=100;
 set timestamp=1, timestamp=default;
-set tmp_table_size=1024;
+set tmp_table_size=16384;
 set tx_isolation="READ-COMMITTED";
 set wait_timeout=100;
 set log_warnings=1;

mysql-test/main/variables.test

@@ -340,7 +340,7 @@ set global table_open_cache=100;
 set default_storage_engine=myisam;
 set global thread_cache_size=100;
 set timestamp=1, timestamp=default;
-set tmp_table_size=1024;
+set tmp_table_size=16384;
 set tx_isolation="READ-COMMITTED";
 set wait_timeout=100;
 set log_warnings=1;

mysql-test/suite/heap/heap.result

@@ -798,58 +798,6 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 3	DERIVED	t1	range	PRIMARY	PRIMARY	4	NULL	1	Using index condition; Using where
 DROP TABLE t1,t2,h1;
 DROP VIEW v1;
-CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=100;
-insert into t1 values(1);
-select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
-data_length	index_length
-1600	2400
-drop table t1;
-CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=10000;
-insert into t1 values(1);
-select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
-data_length	index_length
-16000	24000
-drop table t1;
-CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=3000 max_rows=3000;
-insert into t1 values(1);
-select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
-data_length	index_length
-48000	72000
-drop table t1;
-CREATE TABLE t1 (a int, index(a)) engine=heap max_rows=15000;
-insert into t1 values(1);
-select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
-data_length	index_length
-24000	36000
-drop table t1;
-create table t1 (c1 int, index(c1)) engine=heap max_rows=10000;
-insert into t1 select rand(100000000);
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1 limit 488;
-select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
-data_length	index_length
-16000	24000
-insert into t1 select rand(100000000) from t1 limit 1;
-select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
-data_length	index_length
-33024	49024
-insert into t1 select rand(100000000) from t1 limit 1000;
-select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
-data_length	index_length
-49024	73024
-insert into t1 select rand(100000000) from t1;
-select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
-data_length	index_length
-81024	121024
-drop table t1;
 CREATE TABLE t1 (id INT);
 INSERT INTO t1 VALUES (1);
 INSERT INTO t1 VALUES (2);

mysql-test/suite/heap/heap.test

@@ -546,7 +546,7 @@ CREATE TABLE t1 (
 );
 
 INSERT INTO t1 VALUES (19,5,'h'),(20,5,'h');
-  
+
 CREATE TABLE t2 (col_int_nokey INT);
 
 INSERT INTO t2 VALUES (1),(2);
@@ -567,58 +567,6 @@ DROP TABLE t1,t2,h1;
 DROP VIEW v1;
 # End of 5.1 tests
 
-#
-# Show that MIN_ROWS and MAX_ROWS have an effect on how data_length
-# and index_length are allocated.
-# Result is different for 32 / 64 bit machines as pointer lengths are different
-#
-
-CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=100;
-insert into t1 values(1);
---replace_result 800 1600 1200 2400
-select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
-drop table t1;
-CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=10000;
-insert into t1 values(1);
---replace_result 8000 16000 12000 24000
-select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
-drop table t1;
-CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=3000 max_rows=3000;
-insert into t1 values(1);
---replace_result 24000 48000 36000 72000
-select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
-drop table t1;
-CREATE TABLE t1 (a int, index(a)) engine=heap max_rows=15000;
-insert into t1 values(1);
---replace_result 12000 24000 18000 36000
-select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
-drop table t1;
-
-create table t1 (c1 int, index(c1)) engine=heap max_rows=10000;
-insert into t1 select rand(100000000);
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1;
-insert into t1 select rand(100000000) from t1 limit 488;
---replace_result 8000 16000 12000 24000
-select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
-insert into t1 select rand(100000000) from t1 limit 1;
---replace_result 16512 33024 24512 49024
-select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
-insert into t1 select rand(100000000) from t1 limit 1000;
---replace_result 24512 49024 36512 73024
-select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
-insert into t1 select rand(100000000) from t1;
---replace_result 40512 81024 60512 121024
-select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
-drop table t1;
-
 #
 # MDEV-5905 Creating tmp. memory table kills the server
 #

mysql-test/suite/heap/heap_memory_used,32bit.rdiff

@@ -0,0 +1,40 @@
+--- suite/heap/heap_memory_used.result
++++ suite/heap/heap_memory_used.reject
+@@ -17,13 +17,13 @@
+ insert into t1 values(1);
+ select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+ data_length	index_length
+-65504	131040
++32736	65504
+ drop table t1;
+ CREATE TABLE t1 (a int, index(a)) engine=heap max_rows=15000;
+ insert into t1 values(1);
+ select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+ data_length	index_length
+-16352	32736
++16352	16352
+ drop table t1;
+ create table t1 (c1 int, index(c1)) engine=heap max_rows=10000;
+ insert into t1 select rand(100000000);
+@@ -39,17 +39,17 @@
+ insert into t1 select rand(100000000) from t1 limit 488;
+ select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+ data_length	index_length
+-32704	32704
++16352	16352
+ insert into t1 select rand(100000000) from t1 limit 1;
+ select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+ data_length	index_length
+-32704	32704
++16352	16352
+ insert into t1 select rand(100000000) from t1 limit 1000;
+ select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+ data_length	index_length
+-49056	65408
++32704	32704
+ insert into t1 select rand(100000000) from t1;
+ select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+ data_length	index_length
+-81760	114464
++49056	65408
+ drop table t1;

mysql-test/suite/heap/heap_memory_used.result

@@ -0,0 +1,55 @@
+#
+# Test of heap table memory usage
+#
+CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=100;
+insert into t1 values(1);
+select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+data_length	index_length
+16352	16352
+drop table t1;
+CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=10000;
+insert into t1 values(1);
+select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+data_length	index_length
+16352	16352
+drop table t1;
+CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=3000 max_rows=3000;
+insert into t1 values(1);
+select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+data_length	index_length
+65504	131040
+drop table t1;
+CREATE TABLE t1 (a int, index(a)) engine=heap max_rows=15000;
+insert into t1 values(1);
+select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+data_length	index_length
+16352	32736
+drop table t1;
+create table t1 (c1 int, index(c1)) engine=heap max_rows=10000;
+insert into t1 select rand(100000000);
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1 limit 488;
+select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+data_length	index_length
+32704	32704
+insert into t1 select rand(100000000) from t1 limit 1;
+select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+data_length	index_length
+32704	32704
+insert into t1 select rand(100000000) from t1 limit 1000;
+select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+data_length	index_length
+49056	65408
+insert into t1 select rand(100000000) from t1;
+select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+data_length	index_length
+81760	114464
+drop table t1;

mysql-test/suite/heap/heap_memory_used.test

@@ -0,0 +1,50 @@
+--echo #
+--echo # Test of heap table memory usage
+--echo #
+
+--source include/word_size.inc
+
+#
+# Show that MIN_ROWS and MAX_ROWS have an effect on how data_length
+# and index_length are allocated.
+# Result is different for 32 / 64 bit machines as pointer lengths are different
+#
+
+CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=100;
+insert into t1 values(1);
+select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+drop table t1;
+
+CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=10000;
+insert into t1 values(1);
+select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+drop table t1;
+CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=3000 max_rows=3000;
+insert into t1 values(1);
+select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+drop table t1;
+CREATE TABLE t1 (a int, index(a)) engine=heap max_rows=15000;
+insert into t1 values(1);
+select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+drop table t1;
+
+create table t1 (c1 int, index(c1)) engine=heap max_rows=10000;
+insert into t1 select rand(100000000);
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1;
+insert into t1 select rand(100000000) from t1 limit 488;
+select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+insert into t1 select rand(100000000) from t1 limit 1;
+select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+insert into t1 select rand(100000000) from t1 limit 1000;
+select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+insert into t1 select rand(100000000) from t1;
+select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
+drop table t1;

mysql-test/suite/maria/aria_sort_buffer.result

@@ -5,6 +5,8 @@ Note	1105	Cast to unsigned converted negative integer to it's positive complemen
 Note	1105	Cast to unsigned converted negative integer to it's positive complement
 Warning	1292	Truncated incorrect aria_sort_buffer_size value: '18446744073709551615'
 SET SESSION tmp_table_size=65535;
+Warnings:
+Warning	1292	Truncated incorrect tmp_table_size value: '65535'
 CREATE TABLE t1 (a VARCHAR(255));
 insert into t1 (a) select seq from seq_1_to_1000;
 UPDATE t1 SET a=( (SELECT MAX(a) FROM t1));

mysql-test/suite/maria/repair-big-sort.result

@@ -2,6 +2,8 @@ SET sql_mode='';
 CREATE TEMPORARY TABLE t1 (a tinyINT,b CHAR(1)) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
 INSERT INTO t1 VALUES (1,1),(3,3),(2,2);
 SET SESSION tmp_table_size=True;
+Warnings:
+Warning	1292	Truncated incorrect tmp_table_size value: '1'
 CREATE TABLE t2 (c INT, d DATE) ENGINE=InnoDB PARTITION BY RANGE (YEAR (d)) SUBPARTITION BY HASH (TO_DAYS (d)) (PARTITION p0 VALUES LESS THAN (1990) (SUBPARTITION s0, SUBPARTITION s1), PARTITION p1 VALUES LESS THAN MAXVALUE (SUBPARTITION s4, SUBPARTITION s5));
 SET SESSION aria_sort_buffer_size=CAST(-1 AS UNSIGNED INT);
 Warnings:

mysql-test/suite/sys_vars/r/maximum_basic.result

@@ -9,7 +9,7 @@ Warnings:
 Warning	1292	Truncated incorrect tmp_table_size value: '40960'
 SELECT @@session.tmp_table_size;
 @@session.tmp_table_size
-8192
+16384
 SET @@session.max_join_size=40960;
 Warnings:
 Warning	1292	Truncated incorrect max_join_size value: '40960'

mysql-test/suite/sys_vars/r/sysvars_server_embedded.result

@@ -3718,7 +3718,7 @@ VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	If an internal in-memory temporary table exceeds this size, MariaDB will automatically convert it to an on-disk MyISAM or Aria table. Same as tmp_table_size.
 NUMERIC_MIN_VALUE	0
 NUMERIC_MAX_VALUE	18446744073709551615
-NUMERIC_BLOCK_SIZE	1
+NUMERIC_BLOCK_SIZE	16384
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
@@ -3728,7 +3728,7 @@ VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	Alias for tmp_memory_table_size. If an internal in-memory temporary table exceeds this size, MariaDB will automatically convert it to an on-disk MyISAM or Aria table.
 NUMERIC_MIN_VALUE	0
 NUMERIC_MAX_VALUE	18446744073709551615
-NUMERIC_BLOCK_SIZE	1
+NUMERIC_BLOCK_SIZE	16384
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED

mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result

@@ -4498,7 +4498,7 @@ VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	If an internal in-memory temporary table exceeds this size, MariaDB will automatically convert it to an on-disk MyISAM or Aria table. Same as tmp_table_size.
 NUMERIC_MIN_VALUE	0
 NUMERIC_MAX_VALUE	18446744073709551615
-NUMERIC_BLOCK_SIZE	1
+NUMERIC_BLOCK_SIZE	16384
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
@@ -4508,7 +4508,7 @@ VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	Alias for tmp_memory_table_size. If an internal in-memory temporary table exceeds this size, MariaDB will automatically convert it to an on-disk MyISAM or Aria table.
 NUMERIC_MIN_VALUE	0
 NUMERIC_MAX_VALUE	18446744073709551615
-NUMERIC_BLOCK_SIZE	1
+NUMERIC_BLOCK_SIZE	16384
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED

mysql-test/suite/sys_vars/r/tmp_disk_table_size_func.result

@@ -3,7 +3,7 @@
 
 SET @start_tmp_memory_table_size=@@session.tmp_memory_table_size;
 SET @start_tmp_disk_table_size=@@session.tmp_disk_table_size;
-set @@session.tmp_memory_table_size=1000;
+set @@session.tmp_memory_table_size=16384;
 set @@session.tmp_disk_table_size=3000000;
 create table t1 (a int primary key, b varchar(2000));
 insert into t1 select seq,repeat(char(mod(seq,62)+64),seq) from seq_1_to_2000;

mysql-test/suite/sys_vars/r/transaction_alloc_block_size_basic.result

@@ -168,7 +168,6 @@ SET @@global.transaction_alloc_block_size = @start_global_value;
 SELECT @@global.transaction_alloc_block_size;
 @@global.transaction_alloc_block_size
 8192
-SET @@session.tmp_table_size = @start_session_value;
 SELECT @@session.transaction_alloc_block_size;
 @@session.transaction_alloc_block_size
 1024

mysql-test/suite/sys_vars/t/maximum_basic-master.opt

@@ -1,5 +1,5 @@
 --maximum-auto-increment-increment=8192
---maximum-tmp-table-size=8192
+--maximum-tmp-table-size=16384
 --maximum-max-join-size=8192
 --maximum-use-stat-tables=COMPLEMENTARY
 --maximum-sql-mode='REAL_AS_FLOAT,ANSI_QUOTES'

mysql-test/suite/sys_vars/t/tmp_disk_table_size_func.test

@@ -12,7 +12,7 @@
 SET @start_tmp_memory_table_size=@@session.tmp_memory_table_size;
 SET @start_tmp_disk_table_size=@@session.tmp_disk_table_size;
 
-set @@session.tmp_memory_table_size=1000;
+set @@session.tmp_memory_table_size=16384;
 set @@session.tmp_disk_table_size=3000000;
 
 --disable_ps2_protocol

mysql-test/suite/sys_vars/t/transaction_alloc_block_size_basic.test

@@ -218,11 +218,9 @@ SELECT transaction_alloc_block_size = @@session.transaction_alloc_block_size;
 
 SET @@global.transaction_alloc_block_size = @start_global_value;
 SELECT @@global.transaction_alloc_block_size;
-SET @@session.tmp_table_size = @start_session_value;
 SELECT @@session.transaction_alloc_block_size;
 
 
 #############################################################
 #             END OF transaction_alloc_block_size TESTS     #
 #############################################################
-

mysql-test/suite/versioning/r/partition.result

@@ -1117,7 +1117,7 @@ f varchar(45000)
 partition by system_time interval 1 year (partition p1 history,
 partition pn current);
 # fill the table until full
-insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
+insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
 insert into t1 (f) select f from t1;
 ERROR HY000: The table 't1' is full
 # leave space for exactly one record in current partition
@@ -1133,7 +1133,7 @@ f varchar(45000)
 ) with system versioning engine=memory
 partition by system_time interval 1 year (partition p1 history,
 partition pn current);
-insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
+insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
 select * into outfile 'load.data' from t1;
 load data infile 'load.data' replace into table t1;
 load data infile 'load.data' replace into table t1;

mysql-test/suite/versioning/t/partition.test

@@ -977,7 +977,7 @@ create or replace table t1 (
                                             partition pn current);
 
 --echo # fill the table until full
-insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
+insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
 --error ER_RECORD_FILE_FULL
 insert into t1 (f) select f from t1;
 --echo # leave space for exactly one record in current partition
@@ -995,7 +995,7 @@ create or replace table t1 (
   partition by system_time interval 1 year (partition p1 history,
                                             partition pn current);
 
-insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
+insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
 
 --disable_cursor_protocol
 --disable_ps2_protocol

sql/sql_select.cc

@@ -20577,7 +20577,7 @@ bool Create_tmp_table::finalize(THD *thd,
                                  MY_MIN(thd->variables.tmp_memory_table_size,
                                      thd->variables.max_heap_table_size) :
                                  thd->variables.tmp_disk_table_size) /
-                                share->reclength);
+                                MY_ALIGN(share->reclength, sizeof(char*)));
   set_if_bigger(share->max_rows,1);		// For dummy start options
   /*
     Push the LIMIT clause to the temporary table creation, so that we

sql/sys_vars.cc

@@ -4218,7 +4218,7 @@ static Sys_var_ulonglong Sys_tmp_table_size(
        "will automatically convert it to an on-disk MyISAM or Aria table.",
        SESSION_VAR(tmp_memory_table_size), CMD_LINE(REQUIRED_ARG),
        VALID_RANGE(0, (ulonglong)~(intptr)0), DEFAULT(16*1024*1024),
-       BLOCK_SIZE(1));
+       BLOCK_SIZE(16384));
 
 static Sys_var_ulonglong Sys_tmp_memory_table_size(
        "tmp_memory_table_size",
@@ -4227,7 +4227,7 @@ static Sys_var_ulonglong Sys_tmp_memory_table_size(
        "Same as tmp_table_size.",
        SESSION_VAR(tmp_memory_table_size), CMD_LINE(REQUIRED_ARG),
        VALID_RANGE(0, (ulonglong)~(intptr)0), DEFAULT(16*1024*1024),
-       BLOCK_SIZE(1));
+       BLOCK_SIZE(16384));
 
 static Sys_var_ulonglong Sys_tmp_disk_table_size(
        "tmp_disk_table_size",

storage/heap/ha_heap.cc

@@ -619,7 +619,7 @@ static int heap_prepare_hp_create_info(TABLE *table_arg, bool internal_table,
     case HA_KEY_ALG_UNDEF:
     case HA_KEY_ALG_HASH:
       keydef[key].algorithm= HA_KEY_ALG_HASH;
-      mem_per_row+= sizeof(char*) * 2; // = sizeof(HASH_INFO)
+      mem_per_row+= sizeof(HASH_INFO);
       break;
     case HA_KEY_ALG_BTREE:
       keydef[key].algorithm= HA_KEY_ALG_BTREE;
@@ -688,7 +688,6 @@ static int heap_prepare_hp_create_info(TABLE *table_arg, bool internal_table,
       }
     }
   }
-  mem_per_row+= MY_ALIGN(MY_MAX(share->reclength, sizeof(char*)) + 1, sizeof(char*));
   if (table_arg->found_next_number_field)
   {
     keydef[share->next_number_index].flag|= HA_AUTO_KEY;
@@ -696,11 +695,18 @@ static int heap_prepare_hp_create_info(TABLE *table_arg, bool internal_table,
   }
   hp_create_info->auto_key= auto_key;
   hp_create_info->auto_key_type= auto_key_type;
-  hp_create_info->max_table_size=current_thd->variables.max_heap_table_size;
+  hp_create_info->max_table_size= MY_MAX(current_thd->variables.max_heap_table_size, sizeof(HP_PTRS));
   hp_create_info->with_auto_increment= found_real_auto_increment;
   hp_create_info->internal_table= internal_table;
 
-  max_rows= (ha_rows) (hp_create_info->max_table_size / mem_per_row);
+  max_rows= hp_rows_in_memory(share->reclength, mem_per_row,
+                              hp_create_info->max_table_size);
+#ifdef GIVE_ERROR_IF_NOT_MEMORY_TO_INSERT_ONE_ROW
+  /* We do not give the error now but instead give an error on first insert */
+  if (!max_rows)
+    return HA_WRONG_CREATE_OPTION;
+#endif
+
   if (share->max_rows && share->max_rows < max_rows)
     max_rows= share->max_rows;
 

storage/heap/heapdef.h

@@ -100,6 +100,9 @@ extern void hp_clear(HP_SHARE *info);
 extern void hp_clear_keys(HP_SHARE *info);
 extern uint hp_rb_pack_key(HP_KEYDEF *keydef, uchar *key, const uchar *old,
                            key_part_map keypart_map);
+extern ha_rows hp_rows_in_memory(size_t reclength, size_t index_size,
+                          size_t memory_limit);
+extern size_t hp_memory_needed_per_row(size_t reclength);
 
 extern mysql_mutex_t THR_LOCK_heap;
 

storage/heap/hp_block.c

@@ -75,9 +75,11 @@ int hp_get_new_block(HP_SHARE *info, HP_BLOCK *block, size_t *alloc_length)
     Next time we allocate data for X rows.
     When level 1 is full, we allocate data for HP_PTRS_IN_NOD at level 2 and 1
     + X rows at level 0.
-   */
+  */
   *alloc_length= (sizeof(HP_PTRS) * ((i == block->levels) ? i : i - 1) +
                   (ulonglong)block->records_in_block * block->recbuffer);
+  /* Alloc in blocks of powers of 2 */
+  *alloc_length= MY_MAX(*alloc_length, block->alloc_size);
   if (!(root=(HP_PTRS*) my_malloc(hp_key_memory_HP_PTRS, *alloc_length,
                                   MYF(MY_WME |
                                       (info->internal ?

storage/heap/hp_create.c

@@ -15,11 +15,23 @@
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335  USA */
 
 #include "heapdef.h"
+#include <my_bit.h>
 
 static int keys_compare(void *heap_rb, const void *key1, const void *key2);
-static void init_block(HP_BLOCK *block,uint reclength,ulong min_records,
+static void init_block(HP_BLOCK *block, size_t reclength, ulong min_records,
 		       ulong max_records);
 
+
+/*
+  In how many parts are we going to do allocations of memory and indexes
+  If we assigne 1M to the heap table memory, we will allocate roughly
+  (1M/16) bytes per allocaiton
+*/
+static const int heap_allocation_parts= 16;
+
+/* min block allocation */
+static const ulong heap_min_allocation_block= 16384;
+
 /* Create a heap table */
 
 int heap_create(const char *name, HP_CREATE_INFO *create_info,
@@ -170,7 +182,8 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info,
     share->keydef= (HP_KEYDEF*) (share + 1);
     share->key_stat_version= 1;
     keyseg= (HA_KEYSEG*) (share->keydef + keys);
-    init_block(&share->block, visible_offset + 1, min_records, max_records);
+    init_block(&share->block, hp_memory_needed_per_row(reclength),
+               min_records, max_records);
 	/* Fix keys */
     memcpy(share->keydef, keydef, (size_t) (sizeof(keydef[0]) * keys));
     for (i= 0, keyinfo= share->keydef; i < keys; i++, keyinfo++)
@@ -266,44 +279,90 @@ static int keys_compare(void *heap_rb_, const void *key1_,
                     heap_rb->search_flag, not_used);
 }
 
-static void init_block(HP_BLOCK *block, uint reclength, ulong min_records,
+
+/*
+  Calculate length needed for storing one row
+*/
+
+size_t hp_memory_needed_per_row(size_t reclength)
+{
+  /* Data needed for storing record + pointer to records */
+  reclength= MY_MAX(reclength, sizeof(char*));
+  /* The + 1 below is for the delete marker at the end of record*/
+  reclength= MY_ALIGN(reclength+1, sizeof(char*));
+  return reclength;
+}
+
+/*
+  Calculate the number of rows that fits into a given memory size
+*/
+
+ha_rows hp_rows_in_memory(size_t reclength, size_t index_size,
+                          size_t memory_limit)
+{
+  reclength= hp_memory_needed_per_row(reclength);
+  if ((memory_limit < index_size + reclength + sizeof(HP_PTRS)))
+    return 0;                                   /* Wrong arguments */
+  return (ha_rows) ((memory_limit - sizeof(HP_PTRS)) /
+                    (index_size + reclength));
+}
+
+
+static void init_block(HP_BLOCK *block, size_t reclength, ulong min_records,
 		       ulong max_records)
 {
-  ulong i,recbuffer,records_in_block;
+  ulong i,records_in_block;
+  ulong recbuffer= (ulong) MY_ALIGN(reclength, sizeof(uchar*));
+  ulong extra;
+  ulonglong memory_needed;
+  size_t alloc_size;
 
   /*
     If not min_records and max_records are given, optimize for 1000 rows
   */
   if (!min_records)
-    min_records= MY_MIN(1000, max_records);
+    min_records= MY_MIN(1000, max_records / heap_allocation_parts);
   if (!max_records)
     max_records= MY_MAX(min_records, 1000);
+  min_records= MY_MIN(min_records, max_records);
 
-  /*
+ /*
     We don't want too few records_in_block as otherwise the overhead of
     of the HP_PTRS block will be too notable
   */
-  records_in_block= MY_MAX(1000, min_records);
+  records_in_block= MY_MAX(min_records, max_records / heap_allocation_parts);
-  records_in_block= MY_MIN(records_in_block, max_records);
+
-  /* If big max_records is given, allocate bigger blocks */
+  /*
-  records_in_block= MY_MAX(records_in_block, max_records / 10);
+    Align allocation sizes to power of 2 to get less memory fragmentation from
+    system alloc().
+    As long as we have less than 128 allocations, all but one of the
+    allocations will have an extra HP_PTRS size structure at the start
+    of the block.
+
+    We ensure that the block is not smaller than heap_min_allocation_block
+    as otherwise we get strange results when max_records <
+    heap_allocation_parts)
+  */
+  extra= sizeof(HP_PTRS) + MALLOC_OVERHEAD;
+
   /* We don't want too few blocks per row either */
   if (records_in_block < 10)
-    records_in_block= 10;
+    records_in_block= MY_MIN(10, max_records);
+  memory_needed= MY_MAX(((ulonglong) records_in_block * recbuffer + extra),
+                        (ulonglong) heap_min_allocation_block);
+
+  /* We have to limit memory to INT_MAX32 as my_round_up_to_next_power() is 32 bit */
+  memory_needed= MY_MIN(memory_needed, (ulonglong) INT_MAX32);
+  alloc_size= my_round_up_to_next_power((uint32)memory_needed);
+  records_in_block= (ulong) ((alloc_size - extra)/ recbuffer);
+
+  DBUG_PRINT("info", ("records_in_block: %lu" ,records_in_block));
 
-  recbuffer= (uint) (reclength + sizeof(uchar**) - 1) & ~(sizeof(uchar**) - 1);
-  /*
-    Don't allocate more than my_default_record_cache_size per level.
-    The + 1 is there to ensure that we get at least 1 row per level (for
-    the exceptional case of very long rows)
-  */
-  if ((ulonglong) records_in_block*recbuffer >
-      (my_default_record_cache_size-sizeof(HP_PTRS)*HP_MAX_LEVELS))
-    records_in_block= (my_default_record_cache_size - sizeof(HP_PTRS) *
-                       HP_MAX_LEVELS) / recbuffer + 1;
   block->records_in_block= records_in_block;
   block->recbuffer= recbuffer;
   block->last_allocated= 0L;
+  /* All alloctions are done with this size, if possible */
+  block->alloc_size= alloc_size - MALLOC_OVERHEAD;
 
   for (i= 0; i <= HP_MAX_LEVELS; i++)
     block->level_info[i].records_under_level=

storage/heap/hp_write.c

@@ -145,21 +145,22 @@ static uchar *next_free_record_pos(HP_SHARE *info)
     DBUG_PRINT("exit",("Used old position: %p", pos));
     DBUG_RETURN(pos);
   }
-  if ((info->records > info->max_records && info->max_records) ||
-      (info->data_length + info->index_length >= info->max_table_size))
-  {
-    DBUG_PRINT("error",
-                ("record file full. records: %lu  max_records: %lu  "
-                "data_length: %llu  index_length: %llu  "
-                "max_table_size: %llu",
-                info->records, info->max_records,
-                info->data_length, info->index_length,
-                info->max_table_size));
-    my_errno=HA_ERR_RECORD_FILE_FULL;
-    DBUG_RETURN(NULL);
-  }
   if (!(block_pos=(info->records % info->block.records_in_block)))
   {
+    if ((info->records > info->max_records && info->max_records) ||
+        (info->data_length + info->index_length >= info->max_table_size))
+    {
+      DBUG_PRINT("error",
+                 ("record file full. records: %lu  max_records: %lu  "
+                  "data_length: %llu  index_length: %llu  "
+                  "max_table_size: %llu",
+                  info->records, info->max_records,
+                  info->data_length, info->index_length,
+                  info->max_table_size));
+      my_errno=HA_ERR_RECORD_FILE_FULL;
+      DBUG_RETURN(NULL);
+    }
+
     if (hp_get_new_block(info, &info->block,&length))
       DBUG_RETURN(NULL);
     info->data_length+=length;