database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2026-10-23 18:49:29 +03:00
Code Activity

MDEV-40669 HEAP MIN_ROWS pre-sizes blocks past max_heap_table_size

Browse Source 
`init_block()` captures `requested_min_records` from the caller's
`min_records` before the `min_records= MY_MIN(min_records,
max_records)` clamp, then restores that raw value when the block
allocation cap added by MDEV-40447 fires.

`max_records` is derived from `max_heap_table_size` /
`tmp_memory_table_size` and is the most rows the table can ever hold,
so that clamp is what has always bounded a HEAP table's allocations.
Restoring the pre-clamp value undoes it, and an unreachable `MIN_ROWS`
pre-sizes the record block and every hash key block past the ceiling.
With `max_heap_table_size=64M` and `MIN_ROWS=20000000` a one-row table
allocates 1GB where it used to allocate 96MB; `MIN_ROWS=4294967295` at
the shipped default 16MB ceiling allocates 4GB, bounded only by the
`INT_MAX32` clamp on `memory_needed`.  A few such tables exhaust
memory.

Fix: clamp `requested_min_records` to `max_records` as well.  `MY_MIN`
keeps 0 at 0, so "no `min_records` requested" stays distinguishable
from an explicit `MIN_ROWS`, and the cap keeps ignoring the defaulted
1000-row heuristic.

The cap and the clamp together give four regimes, and only the last
one changes:

1. No `MIN_ROWS`: capped, sizing comes from the ceiling alone.
2. `MIN_ROWS` below the cap: capped.
3. `MIN_ROWS` above the cap but within `max_records`: pre-sizes to
   `MIN_ROWS`, past the cap, as MDEV-40447 intends.
4. `MIN_ROWS` at or above `max_records`: unreachable, so it degrades
   to plain ceiling-derived sizing.

In case 4 the cap branch becomes a no-op, because `records_in_block`
already equals `max_records`, so sizing returns to exactly what it was
before MDEV-40447.

`init_block()` no longer defaults `max_records` itself.  The block
sizing ceiling is derived once in `heap_create()` and the parameter is
`const`, so the caller's value reaches `share->max_records` unchanged:
0 there means "no row limit", and `hp_alloc_from_tail()` skips the
limit check only while it is 0.

Tests:
- `storage/heap/hp_test_block_size-t.c`: a four-case boundary walk
  asserting the exact `alloc_size` of each regime above at one
  ceiling-derived `max_records`, and a keyed case asserting that an
  unreachable `MIN_ROWS` clamps the hash key block as well as the
  record block (`sizeof(HASH_INFO)` gives that block its own
  `recbuffer` and its own cap).  That one expectation selects on
  `SIZEOF_CHARP`: `sizeof(HASH_INFO)` is 24 on LP64 and 12 on ILP32,
  which halves `memory_needed` and rounds a whole power of two lower.
  The record-block sizes round the same on both widths.  A
  `max_records=0` case covers the derived ceiling: the block is sized
  from it while `share->max_records` stays 0, and the table accepts far
  more rows than that default.
- `mysql-test/suite/heap/min_rows_alloc.test`: the same regimes end to
  end across three ceilings, plus `MIN_ROWS` at the .frm maximum under
  the shipped default ceiling.
This commit is contained in:
Arcadiy Ivanov
2026-08-07 14:16:08 -04:00
committed by Monty
parent dd896f05d9
commit e080205915
4 changed files with 399 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
mysql-test/suite/heap/min_rows_alloc.result Normal file
View File

@@ -0,0 +1,89 @@
#
# A ceiling large enough for the block cap to be live.
# (a INT, b INT, KEY(a)) rows are 16 bytes, hash entries 24.
#
SET SESSION max_heap_table_size= 256*1024*1024;
#
# No MIN_ROWS: sizing comes from the ceiling alone and is capped.
#
CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY;
INSERT INTO t1 VALUES (1,1),(2,2);
SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
DATA_LENGTH INDEX_LENGTH
4194272 4194272
DROP TABLE t1;
#
# MIN_ROWS below the cap does not raise the block above the cap.
#
CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=1000;
INSERT INTO t1 VALUES (1,1),(2,2);
SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
DATA_LENGTH INDEX_LENGTH
4194272 4194272
DROP TABLE t1;
#
# MIN_ROWS above the cap but within the ceiling still pre-sizes
# past the cap: it is a real row count expectation.
#
CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=1000000;
INSERT INTO t1 VALUES (1,1),(2,2);
SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
DATA_LENGTH INDEX_LENGTH
16777184 33554400
DROP TABLE t1;
#
# MIN_ROWS beyond the ceiling is unreachable and must clamp to it
# instead of reserving memory the table can never use. Both of
# these hold more rows than 256M can fit, so both must produce the
# same, ceiling-derived, sizing.
#
CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=20000000;
INSERT INTO t1 VALUES (1,1),(2,2);
SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
DATA_LENGTH INDEX_LENGTH
134217696 268435424
DROP TABLE t1;
CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=40000000;
INSERT INTO t1 VALUES (1,1),(2,2);
SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
DATA_LENGTH INDEX_LENGTH
134217696 268435424
DROP TABLE t1;
#
# A smaller ceiling clamps harder, and MIN_ROWS at the .frm
# maximum stays bounded even at the shipped default ceiling.
#
SET SESSION max_heap_table_size= 64*1024*1024;
CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=20000000;
INSERT INTO t1 VALUES (1,1),(2,2);
SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
DATA_LENGTH INDEX_LENGTH
33554400 67108832
DROP TABLE t1;
SET SESSION max_heap_table_size= 16*1024*1024;
CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=4294967295;
INSERT INTO t1 VALUES (1,1),(2,2);
SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
DATA_LENGTH INDEX_LENGTH
8388576 16777184
DROP TABLE t1;
#
# The clamp is a ceiling clamp, not a MIN_ROWS ban: raising the
# ceiling lets the same MIN_ROWS pre-size further.
#
SET SESSION max_heap_table_size= 1024*1024*1024;
CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=8000000;
INSERT INTO t1 VALUES (1,1),(2,2);
SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
DATA_LENGTH INDEX_LENGTH
134217696 268435424
DROP TABLE t1;
SET SESSION max_heap_table_size= DEFAULT;

95
mysql-test/suite/heap/min_rows_alloc.test Normal file
View File

@@ -0,0 +1,95 @@
# MIN_ROWS pre-sizes a MEMORY table's blocks, but it must never pre-size
# them past max_heap_table_size: max_records is derived from that ceiling
# and is the most rows the table can ever hold, so a larger MIN_ROWS is
# unreachable. Block allocations are also capped (see
# main.tmp_table_heap_alloc) when they come from the ceiling rather than
# from a caller row-count expectation. The two interact, so walk all of
# the regimes: no MIN_ROWS, MIN_ROWS below the cap, MIN_ROWS above the
# cap but reachable, and MIN_ROWS beyond the ceiling.
#
# Sizes are exact byte counts and depend on pointer width.
--source include/have_64bit.inc
--echo #
--echo # A ceiling large enough for the block cap to be live.
--echo # (a INT, b INT, KEY(a)) rows are 16 bytes, hash entries 24.
--echo #
SET SESSION max_heap_table_size= 256*1024*1024;
--echo #
--echo # No MIN_ROWS: sizing comes from the ceiling alone and is capped.
--echo #
CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY;
INSERT INTO t1 VALUES (1,1),(2,2);
SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
DROP TABLE t1;
--echo #
--echo # MIN_ROWS below the cap does not raise the block above the cap.
--echo #
CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=1000;
INSERT INTO t1 VALUES (1,1),(2,2);
SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
DROP TABLE t1;
--echo #
--echo # MIN_ROWS above the cap but within the ceiling still pre-sizes
--echo # past the cap: it is a real row count expectation.
--echo #
CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=1000000;
INSERT INTO t1 VALUES (1,1),(2,2);
SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
DROP TABLE t1;
--echo #
--echo # MIN_ROWS beyond the ceiling is unreachable and must clamp to it
--echo # instead of reserving memory the table can never use. Both of
--echo # these hold more rows than 256M can fit, so both must produce the
--echo # same, ceiling-derived, sizing.
--echo #
CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=20000000;
INSERT INTO t1 VALUES (1,1),(2,2);
SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
DROP TABLE t1;
CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=40000000;
INSERT INTO t1 VALUES (1,1),(2,2);
SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
DROP TABLE t1;
--echo #
--echo # A smaller ceiling clamps harder, and MIN_ROWS at the .frm
--echo # maximum stays bounded even at the shipped default ceiling.
--echo #
SET SESSION max_heap_table_size= 64*1024*1024;
CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=20000000;
INSERT INTO t1 VALUES (1,1),(2,2);
SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
DROP TABLE t1;
SET SESSION max_heap_table_size= 16*1024*1024;
CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=4294967295;
INSERT INTO t1 VALUES (1,1),(2,2);
SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
DROP TABLE t1;
--echo #
--echo # The clamp is a ceiling clamp, not a MIN_ROWS ban: raising the
--echo # ceiling lets the same MIN_ROWS pre-size further.
--echo #
SET SESSION max_heap_table_size= 1024*1024*1024;
CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=8000000;
INSERT INTO t1 VALUES (1,1),(2,2);
SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
DROP TABLE t1;
SET SESSION max_heap_table_size= DEFAULT;

37
storage/heap/hp_create.c
View File

@@ -20,7 +20,7 @@
static int keys_compare(void *heap_rb, const void *key1, const void *key2);
static void init_block(HP_BLOCK *block, size_t reclength, ulong min_records,
ulong max_records);
const ulong max_records);
/*
@@ -64,6 +64,15 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info,
ulong min_records= create_info->min_records;
ulong max_records= create_info->max_records;
uint visible_offset;
/*
max_records is the table's row limit and 0 means "no limit"; it is
stored as such in share->max_records, where hp_alloc_from_tail()
relies on 0 disabling the check. Block sizing needs a concrete row
count instead, so derive that ceiling here and leave max_records
itself untouched.
*/
ulong block_max_records= (max_records ? max_records :
MY_MAX(min_records, 1000));
DBUG_ENTER("heap_create");
if (!create_info->internal_table)
@@ -244,7 +253,7 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info,
share->blob_count= create_info->blob_count;
}
init_block(&share->block, hp_memory_needed_per_row(reclength),
min_records, max_records);
min_records, block_max_records);
/* Fix keys */
memcpy(share->keydef, keydef, (size_t) (sizeof(keydef[0]) * keys));
for (i= 0, keyinfo= share->keydef; i < keys; i++, keyinfo++)
@@ -273,7 +282,7 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info,
else
{
init_block(&keyinfo->block, sizeof(HASH_INFO), min_records,
max_records);
block_max_records);
keyinfo->delete_key= hp_delete_key;
keyinfo->write_key= hp_write_key;
keyinfo->hash_buckets= 0;
@@ -376,7 +385,7 @@ ha_rows hp_rows_in_memory(size_t reclength, size_t index_size,
static void init_block(HP_BLOCK *block, size_t reclength, ulong min_records,
ulong max_records)
const ulong max_records)
{
ulong i,records_in_block,cap_records;
ulong recbuffer= (ulong) MY_ALIGN(reclength, sizeof(uchar*));
@@ -386,13 +395,22 @@ static void init_block(HP_BLOCK *block, size_t reclength, ulong min_records,
size_t alloc_size;
/*
If not min_records and max_records are given, optimize for 1000 rows
If no min_records is given, optimize for 1000 rows. max_records is
the caller's sizing ceiling and is never changed here.
*/
if (!min_records)
min_records= MY_MIN(1000, max_records / heap_allocation_parts);
if (!max_records)
max_records= MY_MAX(min_records, 1000);
min_records= MY_MIN(min_records, max_records);
/*
An explicit min_records may override the cap below, but only as far
as the ceiling reaches: max_records is the most rows the table's
memory ceiling (max_heap_table_size / tmp_memory_table_size) can
ever hold, so a larger MIN_ROWS is unreachable and pre-sizing for it
would reserve memory the table can never use. MY_MIN keeps 0 at 0,
so "no min_records requested" stays distinguishable from an explicit
one.
*/
requested_min_records= MY_MIN(requested_min_records, max_records);
/*
We don't want too few records_in_block as otherwise the overhead of
@@ -418,7 +436,10 @@ static void init_block(HP_BLOCK *block, size_t reclength, ulong min_records,
heap_max_allocation_block. Only an explicit min_records from the
caller (CREATE TABLE ... MIN_ROWS) is a real row count expectation
and may pre-size beyond the cap; the 1000-row default min_records
is a heuristic and must not override the cap.
is a heuristic and must not override the cap. That override is
bounded by max_records (clamped above), so an unreachable MIN_ROWS
degrades to plain ceiling-derived sizing instead of reserving a
block the ceiling can never fill.
*/
cap_records= (heap_max_allocation_block - extra) / recbuffer;
if (records_in_block > cap_records)

187
storage/heap/hp_test_block_size-t.c
View File

@@ -16,7 +16,12 @@
that memory allocators recycle without returning pages to the OS;
- an explicit min_records pre-sizing hint (CREATE TABLE ... MIN_ROWS)
still overrides the cap;
- an explicit min_records below the cap does not defeat the cap;
- an explicit min_records above max_records is unreachable and is
clamped to the ceiling instead of pre-sizing past it;
- small tables are sized exactly as before;
- max_records=0 keeps meaning "no row limit" even though block
sizing needs a concrete row count;
- a capped table remains fully functional past the first block.
*/
@@ -407,11 +412,182 @@ static void test_explicit_min_records_wide_row(void)
}
/*
Walk the whole min_records / cap interaction at one ceiling-derived
max_records, so the boundaries between the regimes are pinned down
together rather than one at a time.
max_records 1M with 104-byte rows puts max_records/heap_allocation_parts
(62500 records) above cap_records (40319), so the cap is live for every
case here and only min_records decides the outcome:
- no min_records -> capped (the heuristic must not override)
- min_records below cap -> capped (too small to raise the block)
- cap < min_records < max -> pre-sized to min_records, past the cap
- min_records >= max -> unreachable, clamped to max_records
*/
#define BOUNDARY_MAX_RECORDS 1000000UL
static void test_min_records_cap_boundary(void)
{
static const struct
{
const char *name;
ulong min_records;
size_t expected_alloc;
const char *what;
} cases[]=
{
{ "test_block_bnd_none", 0UL,
4194272, "no min_records is capped" },
{ "test_block_bnd_small", 10000UL,
4194272, "min_records below the cap is capped" },
{ "test_block_bnd_mid", 200000UL,
33554400, "min_records above the cap pre-sizes past it" },
{ "test_block_bnd_over", 100000000UL,
134217696, "min_records above max_records clamps to the ceiling" }
};
uint i;
for (i= 0; i < array_elements(cases); i++)
{
HP_SHARE *share;
HP_INFO *info;
if (create_table(cases[i].name, 0, REC_LENGTH, cases[i].min_records,
BOUNDARY_MAX_RECORDS, NULL, &share))
{
ok(0, "setup failed for %s: %d", cases[i].what, my_errno);
continue;
}
ok(share->block.alloc_size == cases[i].expected_alloc,
"%s (got %zu, expected %zu)", cases[i].what,
share->block.alloc_size, cases[i].expected_alloc);
if ((info= heap_open(cases[i].name, 2)))
{
heap_drop_table(info);
heap_close(info);
}
}
}
/*
Test: the ceiling clamp applies to the hash key block too. An
unreachable MIN_ROWS inflates DATA_LENGTH and INDEX_LENGTH alike,
because both blocks run through init_block(). sizeof(HASH_INFO) is
24 on LP64 and 12 on ILP32, so the key block has its own recbuffer,
its own cap_records and its own expected size -- assert both blocks,
not just the record one.
*/
static void test_min_records_above_max_records_keyed(void)
{
HP_SHARE *share;
HP_INFO *info;
HP_KEYDEF keydef;
HA_KEYSEG keyseg;
init_int_keydef(&keydef, &keyseg);
if (create_table("test_block_bnd_keyed", 1, REC_LENGTH, 100000000UL,
BOUNDARY_MAX_RECORDS, &keydef, &share))
{
ok(0, "setup failed: %d", my_errno);
skip(2, "setup failed");
return;
}
ok(share->block.alloc_size == 134217696,
"record block clamped to the ceiling (got %zu)",
share->block.alloc_size);
/*
1M * sizeof(HASH_INFO) + extra, rounded up to the next power of two:
32MB on LP64, 16MB on ILP32. The record block above rounds to the
same size on both, but halving the key record halves memory_needed,
which lands a whole power of two lower.
*/
ok(share->keydef[0].block.alloc_size ==
(SIZEOF_CHARP == 8 ? 33554400 : 16777184),
"hash key block clamped to the ceiling (got %zu)",
share->keydef[0].block.alloc_size);
if ((info= heap_open("test_block_bnd_keyed", 2)))
{
heap_drop_table(info);
heap_close(info);
}
}
/*
Test: max_records=0 means "no row limit", not "1000 rows".
Block sizing needs a concrete row count, so heap_create() derives one
when the caller passes max_records=0. That derived value must not
reach share->max_records: hp_alloc_from_tail() only skips the row
limit check while max_records is 0, so storing the derived default
would turn an unlimited table into one that reports
HA_ERR_RECORD_FILE_FULL a few blocks in.
*/
static void test_no_max_records_is_unlimited(void)
{
HP_SHARE *share;
HP_INFO *info;
uchar rec[REC_LENGTH];
ulong i;
ulong write_failures= 0;
const ulong rows= 5000;
if (create_table("test_block_nomax", 0, REC_LENGTH, 0, 0, NULL, &share))
{
ok(0, "setup failed: %d", my_errno);
skip(3, "setup failed");
return;
}
ok(1, "created keyless table with max_records=0");
ok(share->max_records == 0,
"max_records stays 0, meaning no row limit (got %lu)",
share->max_records);
ok(share->block.alloc_size == 16352,
"block sized from the derived default (got %zu, expected 16352)",
share->block.alloc_size);
if (!(info= heap_open("test_block_nomax", 2)))
{
ok(0, "heap_open failed: %d", my_errno);
skip(1, "open failed");
return;
}
heap_extra(info, HA_EXTRA_NO_READCHECK);
for (i= 0; i < rows; i++)
{
build_record(rec, (int32) i);
if (heap_write(info, rec))
write_failures++;
}
ok(write_failures == 0,
"wrote %lu rows, well past the derived default, without error "
"(%lu failures)", rows, write_failures);
heap_drop_table(info);
heap_close(info);
}
int main(int argc __attribute__((unused)),
char **argv __attribute__((unused)))
{
MY_INIT("hp_test_block_size");
plan(24);
plan(34);
diag("Test 1: ceiling-derived max_records capped (keyed table)");
test_huge_max_records_capped();
@@ -437,6 +613,15 @@ int main(int argc __attribute__((unused)),
diag("Test 8: explicit min_records still overrides cap for wide rows");
test_explicit_min_records_wide_row();
diag("Test 9: min_records / cap boundary walk");
test_min_records_cap_boundary();
diag("Test 10: unreachable min_records clamps record and key blocks");
test_min_records_above_max_records_keyed();
diag("Test 11: max_records=0 stays unlimited");
test_no_max_records_is_unlimited();
my_end(0);
return exit_status();
}