From c3641cd5136f7bb0b79475b0bcb0b97f20150e14 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 25 Mar 2008 15:53:57 -0300 Subject: [PATCH] Bug#35272: @@global.key_buffer_size = 4294967295 let the server crash When trying to get the requested amount of memory for the keybuffer, the out of memory could be signaled if one of the tentative allocations fail. Later the server would crash (debug assert) when trying to send a ok packet with a error set. The solution is only to signal the error if all tentative allocations for the keybuffer fail. mysql-test/r/key_cache.result: Add test case result for Bug#35272 mysql-test/t/key_cache.test: Add test case for Bug#35272 mysys/mf_keycache.c: Don't set error on my_large_malloc if allocation fails. Set the error if all tentative allocations failed. --- mysql-test/r/key_cache.result | 5 +++++ mysql-test/t/key_cache.test | 25 +++++++++++++++++++++++++ mysys/mf_keycache.c | 4 +++- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/key_cache.result b/mysql-test/r/key_cache.result index 9ada5dc0784..6f30480d6b1 100644 --- a/mysql-test/r/key_cache.result +++ b/mysql-test/r/key_cache.result @@ -368,3 +368,8 @@ Variable_name Value key_cache_block_size 1536 SET GLOBAL key_cache_block_size= @bug28478_key_cache_block_size; DROP TABLE t1; +SET @save_key_buffer = @@global.key_buffer_size; +SET @@global.key_buffer_size = 4294967295; +SET @@global.key_buffer_size = 9223372036854775807; +SET @@global.key_buffer_size = @save_key_buffer; +End of 5.1 tests diff --git a/mysql-test/t/key_cache.test b/mysql-test/t/key_cache.test index 4c14dc96aaa..f588a964584 100644 --- a/mysql-test/t/key_cache.test +++ b/mysql-test/t/key_cache.test @@ -247,3 +247,28 @@ SET GLOBAL key_cache_block_size= @bug28478_key_cache_block_size; DROP TABLE t1; # End of 4.1 tests + +# +# Bug#35272: @@global.key_buffer_size = 4294967295 let the server crash +# + +SET @save_key_buffer = @@global.key_buffer_size; + +# Wee try to force Out Of Memory here. key_buffer_size is ULL, so +# on a 32 bit machine, 4GB is the most we can ask for before the +# server complains about value/variable mismatch. At the off chance +# of one of our 64-bit machines actually offering us 4GB, we also +# accept "no error" (in addition to the expected "out of memory"). +--error 0,ER_OUTOFMEMORY +SET @@global.key_buffer_size = 4294967295; + +# on 32-bit, we get "out of range", on 64-bit, "out of memory". +--error 0,ER_WRONG_ARGUMENTS,ER_OUTOFMEMORY +--disable_warnings +SET @@global.key_buffer_size = 9223372036854775807; +--enable_warnings + +# restore normal value, just in case we got the 4GB or something. +SET @@global.key_buffer_size = @save_key_buffer; + +--echo End of 5.1 tests diff --git a/mysys/mf_keycache.c b/mysys/mf_keycache.c index a03d71f32d8..8001c61a6b9 100644 --- a/mysys/mf_keycache.c +++ b/mysys/mf_keycache.c @@ -102,6 +102,7 @@ */ #include "mysys_priv.h" +#include "mysys_err.h" #include #include "my_static.h" #include @@ -430,7 +431,7 @@ int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size, /* Allocate memory for cache page buffers */ if ((keycache->block_mem= my_large_malloc((size_t) blocks * keycache->key_cache_block_size, - MYF(MY_WME)))) + MYF(0)))) { /* Allocate memory for blocks, hash_links and hash entries; @@ -445,6 +446,7 @@ int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size, if (blocks < 8) { my_errno= ENOMEM; + my_error(EE_OUTOFMEMORY, MYF(0), blocks * keycache->key_cache_block_size); goto err; } blocks= blocks / 4*3;