From b72903fe54cf83c3dcbec7e3532e23bf946249ba Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 3 Dec 2006 17:06:27 +0100 Subject: [PATCH] Maria: * merging changes done to the key cache since May 2006 into Maria * enabling two small enough page cache's unit tests by default * fix to have non-buffered output in unit tests (to not have a false timeout killing in pushbuild) (patch given by Serg) * removing some warnings of gcc -ansi include/lf.h: getting rid of "warning: ISO C does not allow extra `;' outside of a function" (gcc -ansi) mysys/lf_hash.c: getting rid of "warning: ISO C does not allow extra `;' outside of a function" (gcc -ansi) mysys/mf_pagecache.c: Cosmetic changes to minimize the diff with the key cache. #define PAGECACHE_DEBUG_LOG is not needed (just define PAGECACHE_DEBUG if you want) (this change removes "warning: 'pagecache_debug_print' declared `static' but never defined"). Importing changes made to mf_keycache.c since May 2006, into the page cache. Disabling online resizing in the page cache. Fix for "warning: ISO C90 forbids mixed declarations and code". unittest/mysys/Makefile.am: Of the page cache's unit tests, two are small enough to run on pushbuild, renaming them to a -t suffix. unittest/mytap/tap.c: pushbuild kills a test after seeing no output from it for 10 minutes; so we set the mytap framework to not buffer output (patch given by Serg) so that output is seen more frequently and not "all at the end of the test". --- include/lf.h | 14 ++++---- mysys/lf_hash.c | 2 +- mysys/mf_pagecache.c | 66 +++++++++++++++++++++----------------- unittest/mysys/Makefile.am | 15 +++++---- unittest/mytap/tap.c | 2 ++ 5 files changed, 55 insertions(+), 44 deletions(-) diff --git a/include/lf.h b/include/lf.h index c7198ab75b6..ba29b6e9ca5 100644 --- a/include/lf.h +++ b/include/lf.h @@ -80,14 +80,14 @@ void lf_dynarray_destroy(LF_DYNARRAY *array); nolock_wrap(lf_dynarray_value, void *, (LF_DYNARRAY *array, uint idx), - (array, idx)); + (array, idx)) lock_wrap(lf_dynarray_lvalue, void *, (LF_DYNARRAY *array, uint idx), (array, idx), - &array->lock); + &array->lock) nolock_wrap(lf_dynarray_iterate, int, (LF_DYNARRAY *array, lf_dynarray_func func, void *arg), - (array, func, arg)); + (array, func, arg)) /* pin manager for memory allocator, lf_alloc-pin.c @@ -165,15 +165,15 @@ void lf_pinbox_destroy(LF_PINBOX *pinbox); lock_wrap(lf_pinbox_get_pins, LF_PINS *, (LF_PINBOX *pinbox), (pinbox), - &pinbox->pinstack.lock); + &pinbox->pinstack.lock) lock_wrap_void(lf_pinbox_put_pins, (LF_PINS *pins), (pins), - &pins->pinbox->pinstack.lock); + &pins->pinbox->pinstack.lock) lock_wrap_void(lf_pinbox_free, (LF_PINS *pins, void *addr), (pins, addr), - &pins->pinbox->pinstack.lock); + &pins->pinbox->pinstack.lock) /* memory allocator, lf_alloc-pin.c @@ -208,7 +208,7 @@ uint lf_alloc_in_pool(LF_ALLOCATOR *allocator); lock_wrap(lf_alloc_new, void *, (LF_PINS *pins), (pins), - &pins->pinbox->pinstack.lock); + &pins->pinbox->pinstack.lock) /* extendible hash, lf_hash.c diff --git a/mysys/lf_hash.c b/mysys/lf_hash.c index 7e61ef690c6..e13c5f64f54 100644 --- a/mysys/lf_hash.c +++ b/mysys/lf_hash.c @@ -28,7 +28,7 @@ #include #include -LF_REQUIRE_PINS(3); +LF_REQUIRE_PINS(3) /* An element of the list */ typedef struct { diff --git a/mysys/mf_pagecache.c b/mysys/mf_pagecache.c index 4b92f68d9bf..fe2d827ab50 100755 --- a/mysys/mf_pagecache.c +++ b/mysys/mf_pagecache.c @@ -43,9 +43,9 @@ #include #include "my_static.h" #include +#include #include #include -#include /* @@ -86,8 +86,6 @@ #define PAGECACHE_DEBUG_LOG "my_pagecache_debug.log" */ -#define PAGECACHE_DEBUG_LOG "my_pagecache_debug.log" - /* In key cache we have external raw locking here we use SERIALIZED_READ_FROM_CACHE to avoid problem of reading @@ -115,14 +113,6 @@ /* TODO: put it to my_static.c */ my_bool my_disable_flush_pagecache_blocks= 0; -#if defined(MSDOS) && !defined(M_IC80386) -/* we nead much memory */ -#undef my_malloc_lock -#undef my_free_lock -#define my_malloc_lock(A,B) halloc((long) (A/IO_SIZE),IO_SIZE) -#define my_free_lock(A,B) hfree(A) -#endif /* defined(MSDOS) && !defined(M_IC80386) */ - #define STRUCT_PTR(TYPE, MEMBER, a) \ (TYPE *) ((char *) (a) - offsetof(TYPE, MEMBER)) @@ -314,8 +304,8 @@ my_bool info_check_pin(PAGECACHE_BLOCK_LINK *block, enum pagecache_page_pin mode) { struct st_my_thread_var *thread= my_thread_var; - DBUG_ENTER("info_check_pin"); PAGECACHE_PIN_INFO *info= info_find(block->pin_list, thread); + DBUG_ENTER("info_check_pin"); if (info) { if (mode == PAGECACHE_PIN_LEFT_UNPINNED) @@ -372,10 +362,10 @@ my_bool info_check_lock(PAGECACHE_BLOCK_LINK *block, enum pagecache_page_pin pin) { struct st_my_thread_var *thread= my_thread_var; - DBUG_ENTER("info_check_lock"); PAGECACHE_LOCK_INFO *info= (PAGECACHE_LOCK_INFO *) info_find((PAGECACHE_PIN_INFO *) block->lock_list, thread); + DBUG_ENTER("info_check_lock"); switch(lock) { case PAGECACHE_LOCK_LEFT_UNLOCKED: @@ -605,15 +595,9 @@ uint pagecache_fwrite(PAGECACHE *pagecache, next_power(value) is 2 at the power of (1+floor(log2(value))); e.g. next_power(2)=4, next_power(3)=4. */ -static uint next_power(uint value) +static inline uint next_power(uint value) { - uint old_value= 1; - while (value) - { - old_value= value; - value&= value-1; - } - return (old_value << 1); + return (uint) my_round_up_to_next_power((uint32) value) << 1; } @@ -834,15 +818,24 @@ err: The function starts the operation only when all other threads performing operations with the key cache let her to proceed (when cnt_for_resize=0). -*/ + Before being usable, this function needs: + - to receive fixes for BUG#17332 "changing key_buffer_size on a running + server can crash under load" similar to those done to the key cache + - to have us (Sanja) look at the additional constraints placed on + resizing, due to the page locking specific to this page cache. + So we disable it for now. +*/ +#if 0 /* keep disabled until code is fixed see above !! */ int resize_pagecache(PAGECACHE *pagecache, my_size_t use_mem, uint division_limit, uint age_threshold) { int blocks; +#ifdef THREAD struct st_my_thread_var *thread; PAGECACHE_WQUEUE *wqueue; +#endif DBUG_ENTER("resize_pagecache"); if (!pagecache->inited) @@ -909,6 +902,7 @@ finish: pagecache_pthread_mutex_unlock(&pagecache->cache_lock); DBUG_RETURN(blocks); } +#endif /* 0 */ /* @@ -1504,8 +1498,12 @@ static inline void remove_reader(PAGECACHE_BLOCK_LINK *block) { DBUG_ENTER("remove_reader"); BLOCK_INFO(block); +#ifdef THREAD if (! --block->hash_link->requests && block->condvar) pagecache_pthread_cond_signal(block->condvar); +#else + --block->hash_link->requests; +#endif DBUG_VOID_RETURN; } @@ -1515,7 +1513,8 @@ static inline void remove_reader(PAGECACHE_BLOCK_LINK *block) signals on its termination */ -static inline void wait_for_readers(PAGECACHE *pagecache, +static inline void wait_for_readers(PAGECACHE *pagecache + __attribute__((unused)), PAGECACHE_BLOCK_LINK *block) { #ifdef THREAD @@ -1684,7 +1683,6 @@ static PAGECACHE_HASH_LINK *get_hash_link(PAGECACHE *pagecache, { reg1 PAGECACHE_HASH_LINK *hash_link; PAGECACHE_HASH_LINK **start; - PAGECACHE_PAGE page; KEYCACHE_DBUG_PRINT("get_hash_link", ("fd: %u pos: %lu", (uint) file->file, (ulong) pageno)); @@ -1710,6 +1708,7 @@ restart: #ifdef THREAD /* Wait for a free hash link */ struct st_my_thread_var *thread= my_thread_var; + PAGECACHE_PAGE page; KEYCACHE_DBUG_PRINT("get_hash_link", ("waiting")); page.file= *file; page.pageno= pageno; @@ -2053,8 +2052,10 @@ restart: /* Remove the hash link for this page from the hash table */ unlink_hash(pagecache, block->hash_link); /* All pending requests for this page must be resubmitted */ +#ifdef THREAD if (block->wqueue[COND_FOR_SAVED].last_thread) release_queue(&block->wqueue[COND_FOR_SAVED]); +#endif } link_to_file_list(pagecache, block, file, (my_bool)(block->hash_link ? 1 : 0)); @@ -2209,10 +2210,10 @@ my_bool pagecache_lock_block(PAGECACHE *pagecache, BLOCK_INFO(block); while (block->status & BLOCK_WRLOCK) { - DBUG_PRINT("info", ("fail to lock, waiting...")); /* Lock failed we will wait */ #ifdef THREAD struct st_my_thread_var *thread= my_thread_var; + DBUG_PRINT("info", ("fail to lock, waiting...")); add_to_queue(&block->wqueue[COND_FOR_WRLOCK], thread); dec_counter_for_resize_op(pagecache); do @@ -2403,8 +2404,10 @@ static void read_block(PAGECACHE *pagecache, KEYCACHE_DBUG_PRINT("read_block", ("primary request: new page in cache")); /* Signal that all pending requests for this page now can be processed */ +#ifdef THREAD if (block->wqueue[COND_FOR_REQUESTED].last_thread) release_queue(&block->wqueue[COND_FOR_REQUESTED]); +#endif } else { @@ -3210,9 +3213,11 @@ restart: block->status= (BLOCK_READ | (block->status & BLOCK_WRLOCK)); KEYCACHE_DBUG_PRINT("key_cache_insert", ("primary request: new page in cache")); +#ifdef THREAD /* Signal that all pending requests for this now can be processed. */ if (block->wqueue[COND_FOR_REQUESTED].last_thread) release_queue(&block->wqueue[COND_FOR_REQUESTED]); +#endif } } else @@ -3223,11 +3228,9 @@ restart: if (block->status & BLOCK_CHANGED) link_to_file_list(pagecache, block, &block->hash_link->file, 1); } - else - { - if (! (block->status & BLOCK_CHANGED)) + else if (! (block->status & BLOCK_CHANGED)) link_to_changed_list(pagecache, block); - } + if (! (block->status & BLOCK_ERROR)) { bmove512(block->buffer, buff, pagecache->block_size); @@ -3342,9 +3345,11 @@ static void free_block(PAGECACHE *pagecache, PAGECACHE_BLOCK_LINK *block) /* Keep track of the number of currently unused blocks. */ pagecache->blocks_unused++; +#ifdef THREAD /* All pending requests for this page must be resubmitted. */ if (block->wqueue[COND_FOR_SAVED].last_thread) release_queue(&block->wqueue[COND_FOR_SAVED]); +#endif } @@ -3438,12 +3443,14 @@ static int flush_cached_blocks(PAGECACHE *pagecache, if (!last_errno) last_errno= errno ? errno : -1; } +#ifdef THREAD /* Let to proceed for possible waiting requests to write to the block page. It might happen only during an operation to resize the key cache. */ if (block->wqueue[COND_FOR_SAVED].last_thread) release_queue(&block->wqueue[COND_FOR_SAVED]); +#endif /* type will never be FLUSH_IGNORE_CHANGED here */ if (! (type == FLUSH_KEEP || type == FLUSH_FORCE_WRITE)) { @@ -3970,7 +3977,6 @@ static void ___pagecache_pthread_mutex_unlock(pthread_mutex_t *mutex) { KEYCACHE_THREAD_TRACE_END(""); pthread_mutex_unlock(mutex); - return; } diff --git a/unittest/mysys/Makefile.am b/unittest/mysys/Makefile.am index 8dd0416dbac..f7b72255a72 100644 --- a/unittest/mysys/Makefile.am +++ b/unittest/mysys/Makefile.am @@ -12,7 +12,7 @@ LDADD = $(top_builddir)/unittest/mytap/libmytap.a \ # but don't run them by default (for this, a non "-t" suffix is used). # You can run them by hand (how: see "test" target in upper dir). noinst_PROGRAMS = bitmap-t base64-t my_atomic-t \ - mf_pagecache_single_1k-t-big mf_pagecache_single_8k-t-big \ + mf_pagecache_single_1k-t mf_pagecache_single_8k-t \ mf_pagecache_single_64k-t-big \ mf_pagecache_consist_1k-t-big mf_pagecache_consist_64k-t-big \ mf_pagecache_consist_1kHC-t-big mf_pagecache_consist_64kHC-t-big \ @@ -25,11 +25,14 @@ mf_pagecache_single_src = mf_pagecache_single.c $(top_srcdir)/mysys/mf_pagecache mf_pagecache_consist_src = mf_pagecache_consist.c $(top_srcdir)/mysys/mf_pagecache.c test_file.c test_file.h mf_pagecache_common_cppflags = -DEXTRA_DEBUG -DPAGECACHE_DEBUG -DMAIN -mf_pagecache_single_1k_t_big_SOURCES = $(mf_pagecache_single_src) -mf_pagecache_single_8k_t_big_SOURCES = $(mf_pagecache_single_src) +# Those two take reasonable resources (<100 MB) for < 1 minute +mf_pagecache_single_1k_t_SOURCES = $(mf_pagecache_single_src) +mf_pagecache_single_8k_t_SOURCES = $(mf_pagecache_single_src) +mf_pagecache_single_1k_t_CPPFLAGS = $(mf_pagecache_common_cppflags) -DPAGE_SIZE=1024 +mf_pagecache_single_8k_t_CPPFLAGS = $(mf_pagecache_common_cppflags) -DPAGE_SIZE=8192 + +# All others below are big mf_pagecache_single_64k_t_big_SOURCES = $(mf_pagecache_single_src) -mf_pagecache_single_1k_t_big_CPPFLAGS = $(mf_pagecache_common_cppflags) -DPAGE_SIZE=1024 -mf_pagecache_single_8k_t_big_CPPFLAGS = $(mf_pagecache_common_cppflags) -DPAGE_SIZE=8192 mf_pagecache_single_64k_t_big_CPPFLAGS = $(mf_pagecache_common_cppflags) -DPAGE_SIZE=65536 mf_pagecache_consist_1k_t_big_SOURCES = $(mf_pagecache_consist_src) @@ -52,4 +55,4 @@ mf_pagecache_consist_1kWR_t_big_CPPFLAGS = $(mf_pagecache_common_cppflags) -DPA mf_pagecache_consist_64kWR_t_big_SOURCES = $(mf_pagecache_consist_src) mf_pagecache_consist_64kWR_t_big_CPPFLAGS = $(mf_pagecache_common_cppflags) -DPAGE_SIZE=65536 -DTEST_WRITERS -CLEANFILES = my_pagecache_debug.log page_cache_test_file_1 +CLEANFILES = pagecache_debug.log page_cache_test_file_1 diff --git a/unittest/mytap/tap.c b/unittest/mytap/tap.c index 17ec51863d9..51330ad820b 100644 --- a/unittest/mytap/tap.c +++ b/unittest/mytap/tap.c @@ -150,6 +150,8 @@ static signal_entry install_signal[]= { void plan(int const count) { + + setvbuf(tapout, 0, _IONBF, 0); /* provide output at once */ /* Install signal handler */